{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow API自定义损失函数和层次，应用到Keras构建的回归模型上"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n",
      "sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.2\n",
      "numpy 1.18.1\n",
      "pandas 0.25.3\n",
      "sklearn 0.22.1\n",
      "tensorflow 2.0.0\n",
      "tensorflow_core.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "# 导入\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _california_housing_dataset:\n",
      "\n",
      "California Housing dataset\n",
      "--------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      "    :Number of Instances: 20640\n",
      "\n",
      "    :Number of Attributes: 8 numeric, predictive attributes and the target\n",
      "\n",
      "    :Attribute Information:\n",
      "        - MedInc        median income in block\n",
      "        - HouseAge      median house age in block\n",
      "        - AveRooms      average number of rooms\n",
      "        - AveBedrms     average number of bedrooms\n",
      "        - Population    block population\n",
      "        - AveOccup      average house occupancy\n",
      "        - Latitude      house block latitude\n",
      "        - Longitude     house block longitude\n",
      "\n",
      "    :Missing Attribute Values: None\n",
      "\n",
      "This dataset was obtained from the StatLib repository.\n",
      "http://lib.stat.cmu.edu/datasets/\n",
      "\n",
      "The target variable is the median house value for California districts.\n",
      "\n",
      "This dataset was derived from the 1990 U.S. census, using one row per census\n",
      "block group. A block group is the smallest geographical unit for which the U.S.\n",
      "Census Bureau publishes sample data (a block group typically has a population\n",
      "of 600 to 3,000 people).\n",
      "\n",
      "It can be downloaded/loaded using the\n",
      ":func:`sklearn.datasets.fetch_california_housing` function.\n",
      "\n",
      ".. topic:: References\n",
      "\n",
      "    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\n",
      "      Statistics and Probability Letters, 33 (1997) 291-297\n",
      "\n",
      "(20640, 8)\n",
      "(20640,)\n"
     ]
    }
   ],
   "source": [
    "# 导入数据集 房价预测\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "print(housing.DESCR)\n",
    "print(housing.data.shape)\n",
    "print(housing.target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (11610,)\n",
      "(3870, 8) (3870,)\n",
      "(5160, 8) (5160,)\n"
     ]
    }
   ],
   "source": [
    "# 切分数据集\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train_all,x_test,y_train_all,y_test = train_test_split(\n",
    "    housing.data,housing.target,random_state=7)\n",
    "x_train,x_valid,y_train,y_valid = train_test_split(\n",
    "    x_train_all,y_train_all,random_state=11)\n",
    "\n",
    "print(x_train.shape,y_train.shape)\n",
    "print(x_valid.shape,y_valid.shape)\n",
    "print(x_test.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对数据进行归一化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "x_train_scaled = scaler.fit_transform(x_train)\n",
    "x_valid_scaled = scaler.transform(x_valid)\n",
    "x_test_scaled = scaler.transform(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型\n",
    "### 自定义损失函数模型构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 自定义损失函数\n",
    "def customized_mse(y_true,y_pred):\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))\n",
    "\n",
    "# 构建模型\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation='relu',input_shape=x_train.shape[1:]),\n",
    "    keras.layers.Dense(1)\n",
    "])\n",
    "model.summary()\n",
    "\n",
    "# 模型编译，固化模型\n",
    "model.compile(loss=customized_mse,optimizer='adam',\n",
    "              metrics=['mean_squared_error'])\n",
    "\n",
    "# 回调函数\n",
    "callbacks = [keras.callbacks.EarlyStopping(patience=5,min_delta=1e-2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/100\n",
      "11610/11610 [==============================] - 2s 199us/sample - loss: 1.3478 - mean_squared_error: 1.3478 - val_loss: 0.7349 - val_mean_squared_error: 0.7349\n",
      "Epoch 2/100\n",
      "11610/11610 [==============================] - 1s 128us/sample - loss: 0.6155 - mean_squared_error: 0.6155 - val_loss: 0.5534 - val_mean_squared_error: 0.5534\n",
      "Epoch 3/100\n",
      "11610/11610 [==============================] - 1s 124us/sample - loss: 0.4769 - mean_squared_error: 0.4769 - val_loss: 0.4723 - val_mean_squared_error: 0.4723\n",
      "Epoch 4/100\n",
      "11610/11610 [==============================] - 1s 125us/sample - loss: 0.4330 - mean_squared_error: 0.4330 - val_loss: 0.4418 - val_mean_squared_error: 0.4418\n",
      "Epoch 5/100\n",
      "11610/11610 [==============================] - 1s 125us/sample - loss: 0.4141 - mean_squared_error: 0.4141 - val_loss: 0.4461 - val_mean_squared_error: 0.4461\n",
      "Epoch 6/100\n",
      "11610/11610 [==============================] - 1s 127us/sample - loss: 0.4049 - mean_squared_error: 0.4049 - val_loss: 0.4256 - val_mean_squared_error: 0.4256\n",
      "Epoch 7/100\n",
      "11610/11610 [==============================] - 1s 126us/sample - loss: 0.3958 - mean_squared_error: 0.3958 - val_loss: 0.4037 - val_mean_squared_error: 0.4037\n",
      "Epoch 8/100\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.3864 - mean_squared_error: 0.3864 - val_loss: 0.3975 - val_mean_squared_error: 0.3975\n",
      "Epoch 9/100\n",
      "11610/11610 [==============================] - 1s 123us/sample - loss: 0.3834 - mean_squared_error: 0.3834 - val_loss: 0.3912 - val_mean_squared_error: 0.3912\n",
      "Epoch 10/100\n",
      "11610/11610 [==============================] - 1s 123us/sample - loss: 0.3818 - mean_squared_error: 0.3818 - val_loss: 0.3856 - val_mean_squared_error: 0.3856\n",
      "Epoch 11/100\n",
      "11610/11610 [==============================] - 1s 123us/sample - loss: 0.3718 - mean_squared_error: 0.3718 - val_loss: 0.3810 - val_mean_squared_error: 0.3810\n",
      "Epoch 12/100\n",
      "11610/11610 [==============================] - 1s 124us/sample - loss: 0.3668 - mean_squared_error: 0.3668 - val_loss: 0.3760 - val_mean_squared_error: 0.3760\n",
      "Epoch 13/100\n",
      "11610/11610 [==============================] - 1s 126us/sample - loss: 0.3681 - mean_squared_error: 0.3681 - val_loss: 0.3717 - val_mean_squared_error: 0.3717\n",
      "Epoch 14/100\n",
      "11610/11610 [==============================] - 1s 126us/sample - loss: 0.3728 - mean_squared_error: 0.3728 - val_loss: 0.3692 - val_mean_squared_error: 0.3692\n",
      "Epoch 15/100\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.3550 - mean_squared_error: 0.3550 - val_loss: 0.3822 - val_mean_squared_error: 0.3822\n",
      "Epoch 16/100\n",
      "11610/11610 [==============================] - 2s 134us/sample - loss: 0.3581 - mean_squared_error: 0.3581 - val_loss: 0.3585 - val_mean_squared_error: 0.3585\n",
      "Epoch 17/100\n",
      "11610/11610 [==============================] - 1s 124us/sample - loss: 0.3491 - mean_squared_error: 0.3491 - val_loss: 0.3593 - val_mean_squared_error: 0.3593\n",
      "Epoch 18/100\n",
      "11610/11610 [==============================] - 1s 124us/sample - loss: 0.3489 - mean_squared_error: 0.3489 - val_loss: 0.3579 - val_mean_squared_error: 0.3579\n",
      "Epoch 19/100\n",
      "11610/11610 [==============================] - 1s 123us/sample - loss: 0.3493 - mean_squared_error: 0.3493 - val_loss: 0.3518 - val_mean_squared_error: 0.3518\n",
      "Epoch 20/100\n",
      "11610/11610 [==============================] - 1s 125us/sample - loss: 0.3542 - mean_squared_error: 0.3542 - val_loss: 0.3517 - val_mean_squared_error: 0.3517\n",
      "Epoch 21/100\n",
      "11610/11610 [==============================] - 1s 122us/sample - loss: 0.3435 - mean_squared_error: 0.3435 - val_loss: 0.3499 - val_mean_squared_error: 0.3499\n"
     ]
    }
   ],
   "source": [
    "# 模型训练\n",
    "history = model.fit(x_train_scaled,y_train,epochs=100,\n",
    "                    validation_data=(x_valid_scaled,y_valid),\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAEzCAYAAAD3t+CnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3xV9f3H8df33H1zk5s9CSSgMsOSIAgytAW3tThQiqvu8XNUa+1PbbWtttW2jtJa+6vaOio4S1ut1oEIoswgS4YQIIPsdW9y5zm/Py7EBBIIkHBzw+f5eNxHcu/5nnM+3xh556zvVxmGgRBCCCFigxbtAoQQQgjRdRLcQgghRAyR4BZCCCFiiAS3EEIIEUMkuIUQQogYIsEthBBCxJBDBrdS6jmlVKVSan0ny5VS6iml1Dal1JdKqbHdX6YQQgghoGtH3C8AZx5k+VnAiXtf1wN/PPqyhBBCCNGRQwa3YRiLgdqDNLkA+JsR8TmQqJTK6q4ChRBCCPGN7rjGnQPsbvO+ZO9nQgghhOhm5m7Yhurgsw7HUVVKXU/kdDp2u/3k/v37H3LjjqZiWjQnxKUfVZHHmq7raFrfvfdP+he7+nLfQPoX6/p6/7Zs2VJtGEba0WyjO4K7BMht874fUNZRQ8MwngWeBRg8eLCxefPmQ258y88L8ZnjGfmjj7qh1GNn0aJFTJs2Ldpl9BjpX+zqy30D6V+s6+v9U0rtPNptdMefNQuBK/beXT4BaDAMo7wbtguAx5aBO1DZXZsTQgghYtohj7iVUn8HpgGpSqkS4CeABcAwjGeAd4CzgW1AM3B1dxYYiMsixbOyOzcphBBCxKxDBrdhGJcdYrkB3NJtFe0vIRtXZQuN9TUkJKb02G6EEEKIWNAd17h7lCUpcvm8pmy7BLcQ4rAEg0FKSkrw+XzRLqXbuN1uNm3aFO0yekxf6Z/dbqdfv35YLJZu33avD+64tAEANFXshGGFUa5GCBFLSkpKiI+PJy8vD6U6egAm9jQ1NREfHx/tMnpMX+ifYRjU1NRQUlJCfn5+t2+/199zn5iZB4CvZld0CxFCxByfz0dKSkqfCW0RG5RSpKSk9NiZnl4f3KlZAwgbCr2+JNqlCCFikIS2iIae/L3r9cFttlipUUmYPB0+Gi6EEL2ay+WKdgmij+n1wQ1QZ07H3rIn2mUIIYQQURcTwe21p+MOyiAsQojYZRgG99xzDyNGjKCgoID58+cDUF5ezpQpUxg9ejQjRozg008/JRwOc9VVV7W2/d3vfhfl6kVv0uvvKgcIxGWT2vQFhq6j+vAYtkKIvuvNN9+kqKiItWvXUl1dTWFhIVOmTOGVV15h5syZ/O///i/hcJjm5maKioooLS1l/fr1ANTX10e5etGbxERwk5CDs8JPQ30N7uSjGptdCHGceuifG9hY1tit2xyWncBPzhvepbZLlizhsssuw2QykZGRwdSpU1mxYgWFhYVcc801BINBvvOd7zB69GgGDhzI9u3bue222zjnnHOYMWNGt9YtYltMHL62HYRFCCFiUWSQyQNNmTKFxYsXk5OTw9y5c/nb3/5GUlISa9euZdq0acybN49rr732GFcrerOYOOKOS49M/9lUuQM4JbrFCCFiUlePjHvKlClT+NOf/sSVV15JbW0tixcv5rHHHmPnzp3k5ORw3XXX4fV6Wb16NWeffTZWq5VZs2YxaNAgrrrqqqjWLnqXmAju5KyBAPiqd0e5EiGEODIXXnghy5YtY9SoUSil+PWvf01mZiZ//etfeeyxx7BYLLhcLv72t79RWlrK1Vdfja7rADz66KNRrl70JjER3CkZuYQMDb2hNNqlCCHEYfF4PEBkQI7HHnuMxx57rN3yK6+8kiuvvPKA9VavXn1M6hOxJyaucZvMZqpVMmYZhEUIIcRxLiaCG6Deko5TBmERQghxnIuZ4PbaM0gIVkW7DCGEECKqYia4g85M0vQqjL03awghhBDHo5gJbtz9sKsg9TUV0a5ECCGEiJqYCW5bSmQQltryHVGuRAghhIiemAnuuLQBADRV7oxyJUIIIUT0xExwJ2fmA+Cv2RXlSoQQQnSHvLw8qquro11GzImd4M7oR8AwoTeURLsUIYQQnQiFQsd0f+Fw+KDvO3Os6+xOMRPcmslEtZaCxVMe7VKEEKLLiouLGTJkCNdeey0jRoxgzpw5fPDBB0yaNIkTTzyR5cuX4/V6ueaaaygsLGTMmDH84x//aF33tNNOY+zYsYwdO5bPPvsMgEWLFjFt2jQuuugihgwZwpw5czqdxATgRz/6EcOGDWPkyJHcfffdAOzYsYOJEydSWFjIAw88gMvlat32ueee27rurbfeygsvvADAww8/TGFhISNGjOD6669v3ee0adP48Y9/zNSpU3nyySepqqpi1qxZFBYWUlhYyNKlSwGoqalhxowZjBkzhhtuuOGgNQO89NJLjB8/ntGjR3PDDTe0hrLL5eLBBx/klFNOYdmyZeTl5fHwww8zefJkXnvtNYqKipgwYQIjR47kwgsvpK6ursM6Y1VMDHm6T4M5DYdPBmERQhyBd38Ee9Z17zYzC+CsXx6y2bZt23jttdd49tlnKSws5JVXXmHJkiUsXLiQRx55hGHDhnH66afz3HPPUV9fz/jx4/nWt75Feno6//3vf7Hb7WzdupXLLruMjz/+GIA1a9awYcMGsrOzmTRpEkuXLmXy5MkH7Lu2tpa33nqLr776CqVU69zet99+OzfddBNXXHEF8+bN61J3b731Vh588EEA5s6dy7/+9S/OO+88IDJn+CeffALA5Zdfzp133snkyZPZtWsXM2fOZNOmTTz00ENMnjyZBx98kH//+988++yzne5r06ZNzJ8/n6VLl2KxWLj55pt5+eWXueKKK/B6vYwYMYKHH364tb3dbmfJkiUAjBw5kqeffpqpU6fy4IMP8tBDD/HEE08cUGesiqng9tozyfKsj3YZQghxWPLz8ykoKABg+PDhnHHGGSilKCgooLi4mJKSEhYuXMjjjz8OgM/nY9euXWRnZ3PrrbdSVFSEyWRiy5YtrdscP348/fr1A2D06NEUFxd3GNwJCQnY7XauvfZazjnnnNaj6aVLl/LGG28AkRC+9957D9mPjz/+mF//+tc0NzdTW1vL8OHDW4P70ksvbW33wQcfsHHjxtb3jY2NNDU1sXjxYt58800AzjnnHJKSkjrd14cffsiqVasoLCwEoKWlhfT0dABMJhOzZs1q137f/hsaGqivr2fq1KlAZCz4iy+++IB2sSymgjvoyiKt8RMMXUdpMXOWXwjRG3ThyLin2Gy21u81TWt9r2kaoVAIk8nEG2+8weDBg9ut99Of/pSMjAzWrl2LruvY7fYOt2kymTq9Zms2m1m+fDkffvghr776Kr///e/56KOPgMjEJx2119sMdOXz+Vq/3nzzzaxcuZLc3Fx++tOfti4DiIuLa/1e13WWLVuGw+E4YPsd7bMjhmFw5ZVXdjgzmt1ux2Qytfus7f4PpqvterOYSj+VkINVhaitkslGhBB9x8yZM3n66adbr/muWbMGiBw9ZmVloWkaL774YpdvvGrL4/HQ0NDA2WefzRNPPEFRUREAkyZN4tVXXwXg5Zdfbm0/YMAANm7ciN/vp6GhgQ8//BD4JsBTU1PxeDy8/vrrne5zxowZ/P73v299v2+fU6ZMad3Xu+++23rtuSNnnHEGr7/+OpWVlUDklP/OnYd+HNjtdpOUlMSnn34KwIsvvth69N1XxFRw21L6AzIIixCib3nggQcIBoOMHDmSESNG8MADDwBw880389e//pUJEyawZcuWIzpabGpq4txzz2XkyJFMnTqV3/3udwA8+eSTzJs3j8LCQhoaGlrb5+bmcskllzBy5EjmzJnDmDFjAEhMTOS6666joKCA73znO62nsDvy1FNPsXLlSkaOHMmwYcN45plnAPjJT37C4sWLGTt2LO+//z79+/fvdBvDhg3j5z//OTNmzGDkyJF8+9vfpry8azcn//Wvf+Wee+5h5MiRFBUVtV6X7yvUoe7q6ymDBw82Nm/efFjrbC36lBPfPpc1p85jzIzv9VBl3WPfXZ99lfQvdvXlvkH7/m3atImhQ4dGt6Bu1tTURHx8fLdv1+Vytc4dHk091b9o6Oj3Tym1yjCMcUez3Zg64k7KzAMgULs7uoUIIYQQURJTN6clp2UTMMwyCIsQQnTgwgsvZMeO9pcSf/WrXzFz5sxDrhuto+2amhrOOOOM1ve6rqNpGh9++CEpKSlRqam3i6ng1kwmqmQQFiGE6NBbb70V7RIOW0pKSuvNa9C3TpX3lJg6VQ5Qb0nH6ZOpPYUQQhyfYi64W+wZJIYqo12GEEIIERUxF9xBVzapeg36ETzPKIQQQsS6mAtuzd0PqwpTWyE3qAkhhDj+xFxw25JzAajdUxzdQoQQogfsm6WrI8XFxZxyyinHsBrRG8VccLvSBwDgrSqObiFCCCFEFMTU42AAKdkDAfDXyCAsQoiu+9XyX/FV7Vfdus0hyUO4d/zBZ9W69957GTBgADfffDMQmThEKcXixYupq6sjGAzy85//nAsuuOCw9u3z+bjppptYuXIlZrOZ3/72t0yfPp0NGzZw9dVXEwgE0HWdN954g+zsbC655BJKSkoIh8M88MADfWKWrONVzAV3YkoGPsMCjTLRiBCi95s9ezZ33HFHa3AvWLCA//znP9x5550kJCRQXV3NhAkTOP/887s8cxbQOof2unXr+Oqrr5gxYwZbtmzhmWee4fbbb2fOnDkEAgHC4TDvvPMO2dnZ/Pvf/wZoNza5iD0xF9xK06jWUrF4JbiFEF13qCPjnjJmzBgqKyspKyujqqqKpKQksrKyuPPOO1m8eDGaplFaWkpFRQWZmZld3u6SJUu47bbbABgyZAgDBgxgy5YtTJw4kV/84heUlJTw3e9+lxNPPJGCggLuvvtu7r33Xs4991xOO+20nuquOAZi7ho3QIMlnTgZhEUIESMuuugiXn/9debPn8/s2bN5+eWXqaqqYtWqVRQVFZGRkdFubuuu6GyCqMsvv5yFCxficDiYOXMmH330ESeddBKrVq2ioKCA++67j4cffrg7uiWiJCaDu9mRSWKwKtplCCFEl8yePZtXX32V119/nYsuuoiGhgbS09OxWCx8/PHHXZpnen9t57besmULu3btYvDgwWzfvp2BAwfyP//zP5x//vl8+eWXlJWV4XQ6+d73vsfdd9/N6tWru7uL4hiKuVPlACFXFqn1NYRDIUzmmOyCEOI4Mnz4cJqamsjJySErK4s5c+Zw3nnnMW7cOEaPHs2QIUMOe5s333wzN954IwUFBZjNZl544QVsNhvz58/npZdewmKxkJmZyYMPPsiKFSu455570DQNi8XCH//4xx7opThWYjL1NHc/zKU6lRW7Sc/Jj3Y5QghxSOvWrWv9PjU1lWXLlnXY7mCzdOXl5fHFF18AYLfbeeGFFw5oc99993Hfffe1+2zmzJldmiFMxIaYPFVuT907CEv59ihXIoQQQhxbXTriVkqdCTwJmID/Mwzjl/stdwMvAf33bvNxwzCe7+ZaW7nS8gBortrVU7sQQoioWbduHXPnzm33mc1maz3aFse3Qwa3UsoEzAO+DZQAK5RSCw3D2Nim2S3ARsMwzlNKpQGblVIvG4YR6ImiU7Mjp8cDtTIIixCi7ykoKGg3R7UQbXXlVPl4YJthGNv3BvGrwP5D/BhAvIqMHuACaoFQt1baRkJSGs2GDRpLe2oXQgghRK/UlVPlOUDbQ9sSYP9R7n8PLATKgHjgUsMw9P03pJS6HrgeIC0tjUWLFh1ByRF5KgW9tviottGTPB5Pr62tO0j/Yldf7hu075/b7aapqSm6BXWzcDjc5/rUVl/qn8/n65H/17oS3B2Nwbf/k/8zgSLgdGAQ8F+l1KeGYTS2W8kwngWeBRg8eLAxbdq0wy54n/XL0kkJ1zP4KLbRkxYtWsTR9K+3k/7Frr7cN2jfv02bNhEfHx/dgrpZU1NTn+tTW32pf3a7nTFjxnT7drtyqrwEyG3zvh+RI+u2rgbeNCK2ATuAw38w8TC0OLJIClX25C6EEEKIXqcrwb0COFEpla+UsgKziZwWb2sXcAaAUioDGAz06LNaYVc2KUYdoWCP3P8mhBBRcbD5uMXBHS8/u0MGt2EYIeBW4D1gE7DAMIwNSqkblVI37m32M+BUpdQ64EPgXsMwqnuqaADlzsGkDKr3yCNhQgjRV4XD4WO2L8Mw0PX2t2d1df/Hss4uPcdtGMY7wDv7ffZMm+/LgBndW9rB2VP7A1BfvoPM3BOO5a6FEDFozyOP4N/UvfNx24YOIfPHPz5om+6cj3vRokXcf//9ZGdnU1RUxHe/+10KCgp48sknaWlp4e2332bQoEFUVVVx4403smtX5MDmiSeeYNKkSSxfvpw77riDlpYWHA4Hzz//PIMHD+aFF15g4cKFNDc38/XXX3PhhRfy61//usMawuEw3//+91m5ciVKKa655hruvPNOVq1axTXXXIPT6WTy5Mm8++67rF+/nhdeeIGVK1fy+9//HoBzzz2Xu+++m2nTpnHTTTexYsUKWlpauOiii3jooYeAyAhx11xzDe+//z633norhYWF3HLLLVRVVeF0Ovnzn//MkCFD2LFjB5dffjmhUIgzzzzzkD+/xx57jAULFuD3+7nwwgt56KGHKC4u5qyzzmL69OksW7aMt99+m+HDh3PXXXfx3nvv8Zvf/Aa/38/dd99NKBSisLCQP/7xj9hstgPqnD179iFr6A4xOXIaQEJGHgDeqsMfnF8IIY6V2bNnM3/+/Nb3CxYs4Oqrr+att95i9erVfPzxx/zgBz/odLav/a1fv54nn3ySdevW8eKLL7JlyxaWL1/Otddey9NPPw3A7bffzp133smKFSt44403uPbaa4HI9J+LFy9mzZo1PPzww/y4zR8dRUVFzJ8/n3Xr1jF//nx27+54nIyioiJKS0tZv34969at4+qrrwbg6quv5qmnnup0KNeO/OIXv2DlypV8+eWXfPLJJ3z55Zety+x2O0uWLGH27Nlcf/31PP3006xatYrHH3+89Y+g22+/vTX8DzUl6vvvv8/WrVtZvnw5RUVFrFq1isWLFwOwefNmrrjiCtasWcOAAQPwer2MGDGCL774gnHjxnHVVVe1/mxCoVC7sd7b1nmsxORY5QDJWZFBWIJ1JVGuRAgRCw51ZNxTuns+7rFjx5KVlQXAoEGDmDEjcrKzoKCAjz/+GIAPPviAjRu/GSOrsbGRpqYmGhoauPLKK9m6dStKKYLBYGubM844A7fbDcCwYcPYuXMnublt70uOGDhwINu3b+e2227jnHPOYcaMGTQ0NFBfX8/UqVMBmDt3Lu++++4h+7JgwQKeffZZQqEQ5eXlbNy4kfz8yL/tl156KRB5vO+zzz7j4osvbl3P7/cDsHTpUt54443Wfd57b+dzrr///vu8//77rXd5ezwetm7dSv/+/RkwYAATJkxobWsymZg1axYQCfX8/HxOOukkAK688krmzZvHHXfc0a7OYylmgzvBnYzXsMsgLEKIXm/ffNx79uw5YD5ui8VCXl5el+fjtlqtrd9rmobNZmv9PhSKjHul6zrLli3D4XC0W/e2225j+vTpvPXWWxQXF7d7LHDfdiASXPu2tb+kpCTWrl3Le++9x7x581iwYAG//e1viYy/dSCz2dzuuvG+fu7YsYPHH3+cFStWkJSUxFVXXdXuZxAXF9fal8TExE5Hkutsv/szDIP77ruPG264od3nxcXFrfvax263YzKZWtc7mP3XPRZi9lS50jSqTWlYveXRLkUIIQ6qJ+bjPpgZM2a0XlMGWkOvoaGBnJwcgA5nFuuK6upqdF1n1qxZ/OxnP2P16tUkJibidrtZsmQJQOs84RC5Xl1UVISu6+zevZvly5cDkbMAcXFxuN1uKioqOj1CT0hIID8/n9deew2IBOnatWsBmDRpEq+++uoB++zIzJkzee6551pnXystLaWy8tCPFA8ZMoTi4mK2bdsGwIsvvth6ZiFaYja4ARqs6bj8FdEuQwghDqqj+bhXrlzJuHHjePnll49oPu6Deeqpp1i5ciUjR45k2LBhPPNM5F7iH/7wh9x3331MmjTpiO+CLi0tZdq0aYwePZqrrrqKRx99FIDnn3+eW265hYkTJ7Y70p80aRL5+fkUFBRw9913M3bsWABGjRrFmDFjGD58ONdccw2TJk3qdJ8vv/wyf/nLXxg1ahTDhw/nH//4BwBPPvkk8+bNo7CwkIaGhoPWPWPGDC6//HImTpxIQUEBF110UZdGaLPb7Tz//PNcfPHFFBQUoGkaN9544yHX60mqqzdEdLfBgwcbmzdvPqptLH/iMvLrl5H20+LuKaobHU+jU/VFfbl/fblvcODIaUOHDo1uQd0sFkYWKy4u5txzz2X9+vWHvW4s9K+rOvr9U0qtMgxj3NFsN6aPuMPxOaQY9QT8Xbs2JIQQQsS6mL05DcCU2A9tt0F1+U6y8wZHuxwhhOgWvWU+7lNOOaX1Du59XnzxRQoKCg66Xl5e3hEdbXeH3vKz60kxHdz2lMijCg0VxRLcQog+o7fMxx2LYddbfnY9KaZPlbtlEBYhxCFE6z4ecXzryd+7mA7ulJyBAIRqOx7hRwhxfLPb7dTU1Eh4i2PKMAxqamqw2+09sv2onSoPVx79wCmuhCQacaKa9p9lVAghoF+/fpSUlFBVVRXtUrqNz+frsUDoDfpK/+x2O/369euRbUctuO0tYdZ+uIBRZ1xyVNup1VJlEBYhRIcsFkvrEJp9xaJFi1qH7eyL+nr/ukPUTpWHNSj987yj3k6jNUMGYRFCCHHciFpw+102BhRVsmP9Z0e1HZ8zk+Rw3zkNJoQQQhxM1ILblJBM2AQb/vDLo9pO2JVNCg34fc3dVJkQQgjRe0UtuDWThV2n5pO7eCs15TuOeDumpMjF/+oyeSRMCCFE3xfVx8GG3HwP1hB88ceHj3gbjpQBANTvOfLwF0IIIWJFVIP7hDHTKR6eTMq/v6ClufGItpGQEQnulupd3VmaEEII0StFfQCW9O9fT4LX4LO/PHJE66flRB71CNbJICxCCCH6vqgH95gz51KWY8e84B3C4dBhr+90uWkgDk0GYRFCCHEciHpwa5qGec4s0quCLH/zj0e0jRotDVuzDMIihBCi74t6cANMnHMXtW4TjS+8eETrN9nScfkru7kqIYQQovfpFcFttTnxfGcq/b9uYv2nbx/2+j5nFikyCIsQQojjQK8IboCJNzxAsw12/unpw15Xj88miUZ8zZ4eqEwIIYToPXpNcCckZ7LnWyPJW1XG7i2rDmtdc+K+QViKe6AyIYQQovfoNcENMOaW+9EVrJ13eI+GOVL7A1BfIYOwCCGE6Nt6VXBnDyxg5/h+5Hy0kbqqrj+X7c6MPMvdUiWDsAghhOjbelVwAwy68S7sQfjimZ91eZ207Ehwh+tLe6osIYQQolfodcE9dMJZ7ByciHvhUgItXZvxy+50UUc8qkmCWwghRN/W64IbIPnqq0hs0ln6t65P+VlrSsMug7AIIYTo43plcI87/zr2ZNrg7wvRdb1L6zTZMoj3V/RwZUIIIUR09crg1jQNY/Z5ZO7xs/Kf/9eldfyODJL16h6uTAghhIiuXhncAJOuvJf6eI3a557vUns9IYdEPLR4m3q4MiGEECJ6em1w2xwu6s87lQGb6/nqi/8csr0lKReAqtKve7o0IYQQImp6bXADTLjpQXwW2PbM7w7Z1pESGYSlsWJnT5clhBBCRE2vDu6ktFxKpw9lwBe7KNu+7qBtE7PyAGiulkFYhBBC9F29OrgBRt58H5oBa/7w84O2S20dhKXkWJQlhBBCREWvD+7+QwopHptF5n+/pLF2T6ftbHYnNbgxySAsQggh+rBeH9wAA278H5x+WPbswY+660yp2Jo7D3chhBAi1sVEcI847TvsGhSP661FBAO+Tts12TJICFQew8qEEEKIYysmghsg/srvkdwQZtnLv+20TcCZRYpedQyrEkIIIY6tmAnuU2bdTGWaheBLr3U6DKqekEMCzXga645xdUIIIcSxETPBbTKZCV58FtmlPoree6nDNpakfgDUlO04lqUJIYQQx0zMBDfAqdfcR2OcouIvz3a4PD7rRAD2fPnhsSxLCCGEOGa6FNxKqTOVUpuVUtuUUj/qpM00pVSRUmqDUuqT7i0zwulKpObs8eStr2HbmkUHLD9p7DQ2WYZz0sanaKiVa91CCCH6nkMGt1LKBMwDzgKGAZcppYbt1yYR+ANwvmEYw4GLe6BWAMbf9AABM3z1x8cOrFXTsJ7/WxKMJr565d6eKkEIIYSImq4ccY8HthmGsd0wjADwKnDBfm0uB940DGMXgGEYPfZMVmr2IHafdgL9l26ncvfmA5YPKpjAyrQLGVf1JtvXf9FTZQghhBBR0ZXgzgF2t3lfsveztk4CkpRSi5RSq5RSV3RXgR0ZdvO9mMKw4g8/63D5kMt+SaOKx/ePuzA6uQNdCCGEiEXmLrRRHXxmdLCdk4EzAAewTCn1uWEYW9ptSKnrgesB0tLSWLRo0WEXvE/dUDf9/rOa/079Fxa764DljWmXcX7Vn/jHsw/hHjL9iPdzpDwez1H1r7eT/sWuvtw3kP7Fur7ev+7QleAuAXLbvO8HlHXQptowDC/gVUotBkYB7YLbMIxngWcBBg8ebEybNu0Iy4ai8B3YbnmI+o0fM+2u3xywPDx5Mlt/+RET9/wN5+V34EpIOuJ9HYlFixZxNP3r7aR/sasv9w2kf7Gur/evO3TlVPkK4ESlVL5SygrMBhbu1+YfwGlKKbNSygmcAmzq3lLbGzn9Ekr6O7G//j6hYOCA5SazGf3MX5NOLev+fn9PliKEEEIcM4cMbsMwQsCtwHtEwniBYRgblFI3KqVu3NtmE/Af4EtgOfB/hmGs77myQdM07HMvJbU2xOcLnuqwzeBxp7M88WxOLvs7u7YU9WQ5QgghxDHRpee4DcN4xzCMkwzDGGQYxi/2fvaMYRjPtL1KXXMAACAASURBVGnzmGEYwwzDGGEYxhM9VXBbE2ffQU2SmZYXX+20zcDZv8anbNS/KTeqCSGEiH0xNXLa/swWK80XfYt+xV7WfrigwzapmblsPOkWRvpWUfTBK8e4QiGEEKJ7xXRwA5x67f147YrSP/+h0zbjLv4hO7QBZCx7CF+z5xhWJ4QQQnSvmA9ulzuFipljGFBUQfGGZR22MVusNJ/xKNlGJWtefegYVyiEEEJ0n5gPboBxN99P2ATr5z3aaZvhk85hVfx0xux8nrLiA0dcE0IIIWJBnwjujAFD2TUxj36Lt1K7Z2en7fpd+ht0NCpeu+sYVieEEEJ0nz4R3ACDb7obWwi++EPnp8Iz+g1i7cBrGeNdwrpP3jyG1QkhhBDdo88E94knn0Hx8GSS//05Lc2NnbYbe+n9lKgs3J/cT8DvO4YVCiGEEEevzwQ3QPp1N5LgNfj4mvMItDR32MZmd1Jz2sP010tZveCRY1yhEEIIcXT6VHCffOZcdn9/BvlFlXxw5Zmdhveo0y+hyDmRgm1/oqqs+NgWKYQQQhyFPhXcADPueZKS688i/8sqPpg7E39Lx89tp130G8yE2fnqD45xhUIIIcSR63PBDfDtu35L6Y3nkr++mo/mzOzwmnfOwOGszp3LuMYP2Pj5f6JQpRBCCHH4+mRwA3zrjscou+UC8jbWsmjOmTR76g9oM/qyh9lDGvb37+1whjEhhBCit+mzwQ1wxm2/pPx/vkv/TXUsnnMW3qbadssdcfGUnfK/DNSLWfXmb6NUpRBCCNF1fTq4AU6/+RdU3nkxuZvr+fTys/E01LRbPmbmlay3jWbopqeorSyNUpVCCCFE1/T54AaYfsPDVP5gNrnbGlg6p314K00j/ju/xWH42PbqvVGsUgghhDi04yK4AaZf9xOq7r6cfl838tllZ9FUX9m6bMDQk1mVeQnjav7F1jWLo1ilEEIIcXDHTXADTPv+A9TcewXZO5pYdtk5NNbuaV02/PJHqFVu9H/fjR4OR7FKIYQQonPHVXADTLnqPup+fDVZOz18cfm5NNSUAxDvTmbH6B8yOLSZlQvnRblKIYQQomPHXXADnDb3hzTefx2Zu7wsv+xc6qsjN6WdfN6NfGUZxglrH6OhrjrKVQohhBAHOi6DG2DS5XfR+OD1ZJY0s3L2edRV7UYzmTCf+ziJRhOb/n5ftEsUQgghDnDcBjfApNl34nnoZtLLW1g9+3zqKndxwqhJrEi9gHEVr7Nj44polyiEEEK0c1wHN8DEi2+j+eFbSd3jY83sC6jds5PBl/0Kj3LS/PZdGLoe7RKFEEKIVsd9cANMmHUL/l/cQUqFj6LLLiAU8LJ52B0MD3zJ6nefi3Z5QgghRCsJ7r3Gf+cG/I/eRUqVn3WXzyJvwvlsMw0id8Uv8DYdOM65EEIIEQ0S3G2MP/86gr/6IUnVfjZ97xJqxt1BOrV8+fcHo12aEEIIAUhwH2DcOVejP3YfiXUBfI88zoeWKZxc+hK7t66NdmlCCCGEBHdHxp51Bfrj/0tCfRDzO8Xs9juofeMHcqOaEEKIqJPg7sTYmd9DPfET4ptClHySSFpDESveelLCWwghRFRJcB/E6DNmY/rdQ8R7dbZ9lEbyF4+y+ZFT2bjs3WiXJoQQ4jglwX0Io864BMtTPyfOB8GFKez8qIHyv9zIsp9PZ/v6L6JdnhBCiOOMBHcXFEybRc7CNyi+fDK2oIXsJU6cr+5h44/n8tY909i19ctolyiEEOI4IcHdRZkDhnHWg39myidFhJ75OcWn5pOxy8SQf1ZQctmlzJ87njUfvx7tMoUQQvRx5mgXEGs0TaNg2iwKps3C3+Jh0QuP4v/nPxm+sgnTigd4L+sh9LOmwZAp0S5VCCFEHyRH3EfB5nAx86ZfcP47XxJ69pesmxyHNRgk77kPyLnvQf512TSWvfY0wYAv2qUKIYToIyS4u8no0y7gkv9bSdYTf2TXBQnsGh4k/asKEh/4A6tOPZl/3XkRXy1/L9plCiGEiHFyqrybDRk3ncFjl7Hwb78jvvhFaqtqqCmOJ/+9DRjv3sFH2XZCZ57G2O/dTmr2oG7dt67rNDfV4a2vxNtYQzjgJ69gEharvVv3I4QQInokuHuA0jTceSczeO4drPnP84xd+TiOUZUsqsglYYdO7nP/pfyF/7J8RCoJF1xAwZlz8Dc34qmvoqWxFl9DHf6mevyN9YQ8TYQ9jYS9XgxvM3hb0Jp9aC1+TC0BLL4QVl8Ym1/HHjDQjPa1rLNCxQlJqDEjyDltJoMnnoXV5ozOD0YIIcRRk+DuQZrJxMnnXEvw23NZ/faTTN04j9Tcet4LjsZb5yJ7eTHun/2Fsp/95Zt1AOfeV1u6Ap9V4bNrBG0mgg4LYYeVQEo8HocdXE6U04HJ5cLsSsAcH4+hG3jXrMK1ficZL38KL3/Keuv9EuRCCBHDJLiPAYvVximX/JBmzw18/vqvOHXHc8Sl+Vh+2bcpSSogXFaKKS4esyseW0IitvhE7O4kHAnJuBLTcCak4oxPQtOO/JaEqtJtfPXRmzR8vhTX+uLWIN9guZ89JyTBmOHkTJnJkIlnS5ALIUQvJsF9DDldbiZc9Qj11f/D8td+ypg9r6MaP6IoeSbawLHkjJxGZu6JqKMI6M6k5ZxA2twfwtzI+wOC/JUl8MoSNlgekCAXQoheTII7ChJTM5lw0zPs2fUDdr35IMNrPySu7t+wCipJZrdrJMHsQlKGTiFv+ClYrLZur0GCXAghYpMEdxRl9j+RzDv+TigYYNvGFdR89SmmkuX0a/qSzC2LYMtjNL9tY4ttCI1pY3GeMIm80dNxJ6V2ey2HE+QVg5Koyc9mo9XD4FPOxGSSXyMhhDhW5F/cXsBssXLCqEmcMGpS62cVJV9T8uUigjuWkVK7hsKSv2IufR59kWKHqT+ViaPQ+k8ge+Q0svOGdvvp9f2DvLrsa7766E3qP1+Ka90Oxn61Ad69h9WOH1I5NAPbhPGcNOMi+g8p7NY6hBBCtCfB3Utl9BtERr9BwPcB8DbV89XaxTRtXUpcxSqG1n5AQu1CKIJqEtkdNwJ/ViGJQ05jYMEkrLbufXY7NXsQk793D3zvHgD+9caLxNVux7vsM5LXl5C0eiHePyzk02QzDQUDSJh0GiNmzCY5c0C31iGEEMc7Ce4YERefyIjJ58Pk8wHQw2F2fLWSyg2L0UqXk924lpxtS2Db7/D/08Im60k0pIxGJedjT80lPj2flKw8EpLSuuXo3JWSy7RZc+G6yMAv29d9yvYP3ib0xSqyPv8a5ydfU/7IC6zp58A35iTSp3yLEWdcgsOZcNT7FkKI41mXglspdSbwJGAC/s8wjF920q4Q+By41DAMmSqrB2kmE/nDTyF/+Cmtn1WX7WTXlx8T2PE5STWrGVv+KtY94XbrNRs2qk2pNFjSaXFkEY7PxpTYD3tKfxIyBpCSPZB4d/Lh1aJpnDBqKieMmgpAMOBj45KFlC56F9OqDfT/91rM/1zLFvNvKB+UCOMKyD39XIZMOFuujwshxGE65L+aSikTMA/4NlACrFBKLTQMY2MH7X4FyIDcUZKaPYDU7KuAqwAIBQNUVOymrnwH3qrdBOt2QUMpVm85Lv8eMuo/J7WuDm13++HWmgwHNaY0Gq0Z+JyZhONzMCXm4EgZgDtzAGk5Bx+q1WK1M+r0Sxh1+iUAeBqqWf/BfKo//Zi4om1k7n2GfLXjR63Xx0/89iwGDB3fAz8VIYToW7pyuDMe2GYYxnYApdSrwAXAxv3a3Qa8AcjdSb2E2WJtc628Y8GAn4ryYur3FOOt2kmorgTVuC/cK8ip3UJKbQPsbL/eaMPF9sWpNNki4a67sjAn5eJIySUhYwCp2fk4XW4AXO5UJsy6BWbdAkDl7s1sfH8+nqWfkbx+N0mrF9K89/p446B0DHc8WmIi5qQk7Mlp2FPScKVm4U7vhzst96gHoxFCiFjWleDOAXa3eV8CnNK2gVIqB7gQOB0J7phisdrIGjCYrAGDO23j9zVTXbaT+j07aKneSbBuNy17tpKkPMT7K8ht2URyTeMB4d5AHDVaGk22dHyOTPS9p+UdKf0ZeNr5pF56O3ZnPDvWL2X7B28R/HwlCV9X4mguJ85nHFBH495XwATeOA1fnIWAy0443oHhdqES3ZgTk7Alp2BPSScuJYuE9BwS03OJT8qQsBdC9AnKMA78B7JdA6UuBmYahnHt3vdzgfGGYdzWps1rwG8Mw/hcKfUC8K+OrnErpa4HrgdIS0s7ecGCBd3Wkd7G4/HgcrmiXUaP2b9/oaAff1M1IU8NNFdhbqnGHqghPliDO1xDql5Dimo8YDsNhpMqlUK9KQWPOYVmWypBeyphRxI6JpRhEPZ5CDXVgacRw9OI5vVi9rRgafZh8/qxN4eIawkT13LgJCut9WngdSianSZ8Tgv+OBvBOAehOAeGKx5c8WguN+b4ZKzxKYRNcSSlZffIKHbRdrz9bvY10r/YNn369FWGYYw7mm105Yi7BMht874fULZfm3HAq0opgFTgbKVUyDCMt9s2MgzjWeBZgMGDBxvTpk07wrJ7v0WLFiH9a8/X4qWmfBcNFcU0V+0kWF+K1liKtbkct7+SfP8OUvwNkcPqNupIoMaURlN8BoGMTPSEoViS+hGXnkdiZh6pWXlYrDbC4RCNNWXUV5bQVF2Gt6ocX20VgdpqQrV1GA2NaI1eTE3NJFU3Y9/ViMurYzpY2Ds1fHFm/C4b4XgnutuFcidgTkrCmpKKIzkdR3IacSkZuFNzcKdm9/qR5eR3M7ZJ/0RXgnsFcKJSKh8oBWYDl7dtYBhG/r7v2xxxtwttIeyOOHIGDiVn4NBO20ROy++gvryY5uqdhOtKUE2l2JvLSPSVkdK8loRqb7t1dENRqZKoM6fhtWcQiMsGdz9sybmkDCogOTOf5Ix+aCbTAfvTdZ3G2nIaKktorCzBW72HltpKyrZsJlHTImHf4MHU1IyzvA77tqoOwz4IVO99tVihxWHC7zQTjLMRdtnRXU5UQjxaQgJmdyK2pBTsSSk4k9OJT8kiITWL+KRMzBbrUf+chRB92yGD2zCMkFLqViJ3i5uA5wzD2KCUunHv8md6uEZxHLHZneQMHE7OwOGdtvE21VNdup3GimJaqncSri/B1FSGo6Wc1OavSWv6HEdFoN06YUPhwY5P2QgoKwFlI6hsBDU7IZOdsGYjbHagmWw4zQ5SM6wkpGSgLCegLA40qxNlc2KyONCtDvyhIL4WD0FvI0Gvh5CnkbCnCb3JA14vytOMydOC2evHWV6PrbkaR4uOLXRgf5r3vsqAFrvC1yb0dacN3WkHVxwqzonJFY85PgFrvBurOxG7OxmnO5W4xFTiEtNwudMk/IXo47r0EK1hGO8A7+z3WYeBbRjGVUdflhCdi4tPJG7IWBgytsPlhq5TX1tJTdl2mip34q/Zhd5Yjgq1oEItaKEWTGEfprAPc9iPLdSERa/GavixGn5s+LEbAWze4OEXZ9/7Svnmo6BhIoSJMCY8hon6kIWmoBlvyIQvqBEIaoSCCj0Ayg9awMDsD2DxBLDXNmELGNj9BnZ/ZL72/YX55sY9iBzx+20afoeZoN1M2Gkj7LRh7P0DoDEY5L0v3sCS4MbmTsaemIIjMYW4pHTikzKIT87AYu3ekff6Mm9TLbvWL6Ni42qav96CUVyCvaKeltxU4qdMpeDsuTKCoOhWMvqF6HOUppGYmkliaiZw6hFtY9GiRUw57TR8LR78Lc34Wzz4W7wEfV5CPi9Bfwthv5dQwIsRDGCEgxh6CMJBjHAw8lUPQjgEehAVDoIeQukhMEKocJA4I0T83s+UEULTg2hGCM0IoxkhnOEm0sKVOJUfgLABzWGNqpCDMiOZRhJoUS7COAALmqFh0RUmfxCaW1DNPkxeH6ZmP44aD1ZfGJtfZ1AAYMsBfd535F9BJPx9do2Aw0zAYSEcZyMc5wCXE+VyYYp3YU5wY01IwpaYjCMptfXI35WUjjM+uc8NrlNd9jW71i2jdvM6fF9vw7S7nISyRpLrI4McZQC6gppkM82pLtK/LCX+85cpf+xlVg6IIzx+JAPO/K5MzCOOmvz2CNEJzWTC6XK3Po8eDYauU1dTQXXpNpr2fE2gZifU7ybOW0qWfw9p4WISaH/N3++0UJmSRr01gxZnPuGEXMzJA4hLzycpeyDrv/qacWNG0FRbgbeukpb6Glrqq/E31BFqbCDU2Iju8UCTF83bgub1Ya1vxlreiL0ljN1vYNYPrDUI1O99wTdH/gG7iaDdQshuQXdYI6f+nXZUXBxaXBwmlwtLfAIWVwK2hETs8Uk49l0CcKcSl5DSLaf/DV2n2duIt7GOlqY6Wjx1oOv0GzIOV0ISAOFwiNKtayjbsJyGLRsI7tiJdXclSXu8uFoMHESej/WboSbDTt1JmTTk5RJ/4lAyho5l4PAJDHdE7ogOBQNsWvpPdv/3H1iXbyB3wTJYsIwVrnupHtUf97TpjDx7Lu6UrKPumzi+SHAL0YspTSMpLYuktCzgtA7bNDXUUl2ylYY9O/BXF2PU7cLqKcHlKyerbimpdfXtnrHPMBTez+xYVRxhzQWmODRzPDaLi7AzAT3JjbLlohyJmJ1uLHFJWF2JOFxJxCWk4IhPRA8F8NRX4K2vwltbRUtDNf76WoJNDYQ8HnSvB93bHDny97ag+fyYmgPYmnxY/LXYfHqnfwBA5I+Ahr0viARlyKwImSFoVoRNipBZETYrwpoibAbdBLoGYc3gjd+Doeloe18mUxizFsZk0jFpBiaTgaYZGIbi6yYLoUYbcQ2KtFodawjcRF6NcYqGDBcV4wdSOzCfxJOGkzPiFE4aWHDIo2azxUrBtFkUTJsFQGXJFja88zLNi5eQuWoncUufZ9ejz1OWH48xcQwDz7yYE8aeLuMNiEOS4BYixsW7k4l3nwLDT+lwua/ZQ2XJNhrKt9NSVUztzg0k2hWmQBPmYCPWkAdXoAqnbwdOw4vLaMasOknUvQKGiZCKI6ycGJoLk9mFzezCQKEsYTR3GJUQOeWvjDCaYaAZGpphQgNMRhjDCKPrOv5QmEBIJxA2CIYgGDIIhyAUAj2kMIIKI6QwdBW5oB9WqDCosELpYAopLC2ghcEUVphDYAqDOQyWEFhCCg0znf1zlwrUug0ak3RKskLY44O4XEBiDnr2yaTlnULuiNNIzT6669Tp/U4i/fqH4PrIeP4bPnmL0v/+E8fKTWS9tJjwS4tZ5jZRNyaf5OnfYuSZc3C5U49qn6JvkuAWoo+zO130P2k0nDQaiFy/n3CQ52QNXce795Ryc2MNvqY6/N56gt46ws0N6L5G8NWj+RsxBZqwhJqwhjzEtUQGWNSVCR0t8nXvK6xZCSozxt73hjJjaCaMtt9rZszKhEmZQIt8hmaOvMw2NLsbk8MdOQvgdGN3JWJ3JRGXEHlZrLbW/k1u0z9d1wkFffhbPARavHu/egj6mgHIGXwyw12JGLpOyfYNVGxcQnjXcpLr15FX+jKWsr/BZ7CHVMpcwwlkjsV94kTyRpyKIy7+iP6bWKx2Rn/7MkZ/+zIAynesZ+M7r+D/9DOyP9uGY9E2tj/8DGUnJKImnswJZ19K/ohJcjQuAAluIcR+lKZF7tyPT4Sc/EOv0MtpmobV5owMjJPYeTulaeSeUEDuCQXATUBk0KCv1y+jfstnWPasJsuzgextn8C23xF6R2ObOZ+axAK03ELSh04m94SCDscLOJSs/BFk3fII3AL+Zg9r/vsKlR++S3zR12S+8CGBFz5kSZKJyiHpeNwJ/HPp37Fa7SjDwNB1jHAYDB1D10Hf99UAPRz5PqyDEflsX5vWl2FElgOYNDCbwWRCmUxgNqPMpsh7sxllMqPMZjSLBWUyoVmsrZ9rFismswVlNmOyWNHMZkxmK5rVSlxiGu70fiSm5mJ19O4BimKBBLcQQnTC7ohjSOG3oPBbrZ9V79lNybpPaSn+gvjqIoZXv4er5m0ogkbi2GkbjCd1JJjtEPKhgi2osK/NY4h+zGEfZt2PRfdhMQKtjyHajAAOFWACQDJwOpT4LHxV5SJQZuOEleXYguXA5i7VrwOGirx0be9XBYZS7T7b917TDUzhvV910HQ6vQ/hcISAmr2vFiu0OE344iwEXXbCCU6MBBeaOwFzYhLVLX4+r90YmWsgLRt3ej8SkrN69GyDHg7j9TTgbaylpbEGPRwmtd+JuJPTemyfR0OCWwghDkNqZi6pmZezbwDJcChE8dYiqjYtRS9ZSWrDOoaV/BWTMggZGj5s+JWVAFYCmp2gshLU7ARMTpqtyegmO2GTHcNsxzA7MCwOMNtRVieaxYFmdZBic6JZnRhorF23nPR4K0ZTBSZfDQ5fJQmhWtL0auI0HyYDNPXNq5pE6kypeGzpBJyRyX4sSZHJftwZA0jNzsPu7HxscF3XCYcChEIBQgEfoaCfcDDQ5mvkez0YIBwKEg76I18DfvRQiHDAh6++Fn9tNcH6WvSGRmhowtTkxdzkw1ntwektx+Ez0Ng7vvbfFwPfPKJYoqDZoWiJM+OPsxKKd6DHx4Hd2lqjbugY4RD63rMMhh65jyJyViFyRgIjjDJ0FHrksUsMNCJfwQAV6bMC0AzCmomQyUrQbMOwOsERjynOjT0+mbjkNKyOODSrDZPFislqw2y1Y7JYMdvsmK12zDYHZqsNi82BxerAYnN0y++gBLcQQhwFk9lM3tBx5A39Zt4Iv68ZXTNhsdpwAd05ZUa9Sul0LO+mhloqy3bQWLkTX81uQvWlmDxl2Jr3kOgrI7l5Le5qL+xov14d8TRoiRitQ/xEEswgcjTe/jPVbt3W90phAjQUlv2WJRIZI1hZDVSqDqmg9oalZmiAIqzrtIQNPH4dXwj8AQgGIBwEww/Kb6D5g1j8QWx7vDiKqzGH2lbWutNWHX1+wGcqcmlDtV3PiNzcGDnbEOabPyGq6arQ3ldPkOAWQohuZrNH5zpu5AmDZBh6cqdtmj0NVJftoLFiJy01uwnVl6A1lWPx1RKJ2r0JZuw9Cm2j/bI279t8/k0wtl2mIq3V3q8oUHu/tv1eKQx/gASbfe9nGuz9w8FQWut29rU3zHZ0azzY3WgONyanG4szEVtcIo74ZBwJycQlJOGMSziimf723dgY8LUQaPFQtaeYml1b8FQVE6wtRXkqMPuqcQQacOpedAPCuiJsKIK6wmPE4VNOQpoDQ3OgmR2w+avDrmN/EtxCCHEccbrc7Z4y6G0WLVrEqb1kdrB2Nza6UyJD146e2mHbgN9H5e5t1JVtpbnia7TancQ37SLeV0ZaqJgkmoDILF1HS4JbCCGEOEpWm51+J4yg3wkjOlzuaayjctcWeGjCUe9LglsIIYToYa6EJFwjOh4k6XDJ0/xCCCFEDJHgFkIIIWKIBLcQQggRQyS4hRBCiBgiwS2EEELEEAluIYQQIoZIcAshhBAxRIJbCCGEiCES3EIIIUQMkeAWQgghYogEtxBCCBFDJLiFEEKIGCLBLYQQQsQQCW4hhBAihkhwCyGEEDFEglsIIYSIIRLcQgghRAyR4BZCCCFiiAS3EEIIEUMkuIUQQogYIsEthBBCxBAJbiGEECKGSHALIYQQMUSCWwghhIghEtxCCCFEDJHgFkIIIWKIBLcQQggRQyS4hRBCiBgiwS2EEELEEAluIYQQIoZIcAshhBAxRIJbCCGEiCES3EIIIUQM6VJwK6XOVEptVkptU0r9qIPlc5RSX+59faaUGtX9pQohhBDikMGtlDIB84CzgGHAZUqpYfs12wFMNQxjJPAz4NnuLlSI/2/v/kPtrus4jj/fbPpHTTKd1cpVGmOxoB9rmJXJRhE6wlVEKGFSyhg4yD+EBoL4r0X9UZhj1chCUiOtEVc0IumPmJhjU8dMpxitrUkWriFkq3d/fL+D09n53nu2e8695332fMDhfM/3+zn3ft57f7/3dc/3nPudJGm4V9yXAQcz88XMfB24D9jUOyAzf5+Z/2gf7gYuHu00JUkSQGTm7AMivgBclZk3tY+vBz6SmVs7xt8KvPfk+L5tm4HNABdddNGHH3jggXlOf3IdP36cZcuWLfY0xsb66prm2sD6qpv2+jZs2PBkZq6bz9dYOsSYGLBuYNpHxAbgRuCKQdszcwftafTVq1fn+vXrh5tlQY899hjWV9c01zfNtYH1VTft9Y3CMMF9CFjZ8/hi4HD/oIh4P/AD4OrMfGU005MkSb2GeY/7CWBVRFwSEecC1wK7egdExDuBB4HrM/O50U9TkiTBEK+4M/NERGwFHgGWADszc39EbGm3bwduBy4EvhcRACfmew5fkiSdaphT5WTmDDDTt257z/JNwCkfRpMkSaPlldMkSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSrE4JYkqRCDW5KkQgxuSZIKMbglSSpkqOCOiKsi4o8RcTAitg3YHhHxnXb7UxGxdvRTlSRJcwZ3RCwB7gKuBtYA10XEmr5hVwOr2ttm4O4Rz1OSJDHcK+7LgIOZ+WJmvg7cB2zqG7MJ+HE2dgPnR8SKEc9VkqSz3jDB/Q7gzz2PD7XrTneMJEmap6VDjIkB6/IMxhARm2lOpQP8KyKeGeL7V7Uc+NtiT2KMrK+uaa4NrK+6aa9v9Xy/wDDBfQhYaG5KrgAABMZJREFU2fP4YuDwGYwhM3cAOwAi4g+Zue60ZluI9dU2zfVNc21gfdWdDfXN92sMc6r8CWBVRFwSEecC1wK7+sbsAr7cfrr8cuDVzDwy38lJkqT/N+cr7sw8ERFbgUeAJcDOzNwfEVva7duBGWAjcBB4DfjK+KYsSdLZa5hT5WTmDE04967b3rOcwM2n+b13nOb4aqyvtmmub5prA+urzvrmEE3mSpKkCrzkqSRJhYw9uKf5cqkRsTIifhsRByJif0R8bcCY9RHxakTsbW+3L8Zcz1REvBQRT7dzP+XTkFX7FxGre3qyNyKORcQtfWNK9S4idkbEy71/ZhkRF0TEryPi+fb+zR3PnfU4nQQd9X0zIp5t972HIuL8jufOuh9Pgo767oiIv/Tsgxs7nlu1f/f31PZSROzteO5E968rC8Z2/GXm2G40H2Z7AbgUOBfYB6zpG7MReJjmb8EvBx4f55xGXN8KYG27fB7w3ID61gO/Wuy5zqPGl4Dls2wv27+eGpYAfwXeVbl3wJXAWuCZnnXfALa1y9uAOzvqn/U4nYRbR32fBpa2y3cOqq/dNut+PAm3jvruAG6d43ll+9e3/VvA7RX715UF4zr+xv2Ke6ovl5qZRzJzT7v8T+AAZ98V48r2r8cngRcy80+LPZH5yMzfAX/vW70JuKddvgf47ICnDnOcLrpB9WXmo5l5on24m+YaEiV19G8YZft3UkQE8EXgpws6qRGZJQvGcvyNO7jPmsulRsS7gQ8Bjw/Y/NGI2BcRD0fE+xZ0YvOXwKMR8WQ0V77rNw39u5buHxiVewfw1myvqdDev2XAmGnoIcBXac7+DDLXfjzJtrZvBezsONU6Df37BHA0M5/v2F6mf31ZMJbjb9zBPbLLpU6yiFgG/By4JTOP9W3eQ3MK9gPAd4FfLPT85unjmbmW5n+AuzkiruzbXrp/0VxU6BrgZwM2V+/dsEr3ECAibgNOAPd2DJlrP55UdwPvAT4IHKE5ndyvfP+A65j91XaJ/s2RBZ1PG7Bu1v6NO7hHdrnUSRUR59A06t7MfLB/e2Yey8zj7fIMcE5ELF/gaZ6xzDzc3r8MPERzWqdX6f7R/CDYk5lH+zdU713r6Mm3Ltr7lweMKd3DiLgB+AzwpWzfNOw3xH48kTLzaGb+JzP/C3yfwfOu3r+lwOeB+7vGVOhfRxaM5fgbd3BP9eVS2/dlfggcyMxvd4x5WzuOiLiM5t/8lYWb5ZmLiDdGxHknl2k+CNT/H8OU7V+r8zf9yr3rsQu4oV2+AfjlgDHDHKcTKSKuAr4OXJOZr3WMGWY/nkh9nxf5HIPnXbZ/rU8Bz2bmoUEbK/RvliwYz/G3AJ+220jzCbsXgNvadVuALe1yAHe1258G1o17TiOs7QqaUxpPAXvb28a++rYC+2k+Kbgb+Nhiz/s06ru0nfe+toZp698baIL4TT3ryvaO5heQI8C/aX6LvxG4EPgN8Hx7f0E79u3ATM9zTzlOJ+3WUd9BmvcHTx5/2/vr69qPJ+3WUd9P2uPqKZof5iumqX/t+h+dPOZ6xpbq3yxZMJbjzyunSZJUiFdOkySpEINbkqRCDG5JkgoxuCVJKsTgliSpEINbkqRCDG5JkgoxuCVJKuR/9sECFJb0ZLkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画图展示变化，打印学习曲线\n",
    "def plot_learning_curves(history):\n",
    "    pd.DataFrame(history.history).plot(figsize=(8,5))\n",
    "    plt.grid(True)\n",
    "    plt.gca().set_ylim(0,1)\n",
    "    plt.show()\n",
    "    \n",
    "plot_learning_curves(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 在测试集上进行指标评估\n",
    "model.evaluate(x_test_scaled,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 自定义层次模型构建\n",
    "#### Dense layer 回顾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=31932, shape=(10, 100), dtype=float32, numpy=\n",
       "array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dense layer 回顾\n",
    "layer = tf.keras.layers.Dense(100)\n",
    "layer = tf.keras.layers.Dense(100, input_shape=(None,5))\n",
    "layer(tf.zeros([10,5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense_3/kernel:0' shape=(5, 100) dtype=float32, numpy=\n",
       " array([[-1.87438279e-01,  1.17057130e-01, -1.99815318e-01,\n",
       "          2.22023234e-01, -1.62328482e-02, -2.33810067e-01,\n",
       "          3.29867154e-02, -1.45707473e-01, -2.31261343e-01,\n",
       "          8.25258344e-02, -3.88459265e-02, -1.85060874e-01,\n",
       "          1.59038469e-01,  2.17499152e-01,  1.54771969e-01,\n",
       "         -1.90375462e-01, -1.96589112e-01, -6.94796443e-04,\n",
       "         -1.38469934e-01, -7.61686116e-02,  1.67824551e-01,\n",
       "         -1.56438157e-01, -1.17947161e-03,  1.35876879e-01,\n",
       "         -1.52204037e-01,  2.02305511e-01,  1.01615176e-01,\n",
       "         -2.21040636e-01,  1.10610083e-01,  1.40476078e-02,\n",
       "         -2.17021137e-01,  2.17639163e-01,  3.83620709e-02,\n",
       "         -7.87134618e-02,  1.23965189e-01, -4.38629091e-02,\n",
       "          1.82106540e-01, -1.87780917e-01, -1.63990647e-01,\n",
       "         -2.54983008e-02, -3.36732566e-02, -1.94116876e-01,\n",
       "         -1.73759341e-01, -1.16186887e-02,  2.13929161e-01,\n",
       "          7.93157667e-02, -1.02152780e-01,  4.30196971e-02,\n",
       "         -1.16929226e-01, -2.57502645e-02, -6.97175860e-02,\n",
       "         -1.77903354e-01,  1.74064323e-01,  1.46787718e-01,\n",
       "         -1.80177033e-01,  9.60052460e-02,  1.66205093e-01,\n",
       "         -1.33879721e-01,  2.24551573e-01, -1.96295545e-01,\n",
       "          9.07173306e-02,  1.78772882e-01,  1.42574117e-01,\n",
       "          9.05241221e-02,  6.89752549e-02, -7.67410547e-02,\n",
       "          1.33813903e-01,  2.33949408e-01, -2.13704497e-01,\n",
       "         -1.65827915e-01,  1.93867818e-01,  7.07826763e-02,\n",
       "         -6.38635606e-02,  1.03244439e-01,  2.27196559e-01,\n",
       "         -1.42239273e-01,  2.61351317e-02, -1.19930021e-01,\n",
       "          6.50626272e-02,  5.25554270e-02, -2.20233828e-01,\n",
       "         -4.81836498e-02, -2.09291309e-01, -1.03912726e-01,\n",
       "          1.46329656e-01,  4.45419699e-02,  8.18864852e-02,\n",
       "         -2.36938074e-01, -9.47593153e-02,  3.00259441e-02,\n",
       "         -9.03265178e-02, -9.28232074e-03,  2.26433575e-03,\n",
       "         -1.97307348e-01,  3.36386710e-02,  4.93441969e-02,\n",
       "         -4.51138467e-02, -7.86980987e-03, -2.14031413e-01,\n",
       "          6.39158487e-03],\n",
       "        [ 1.91099659e-01,  1.19183213e-02, -1.48381293e-01,\n",
       "         -8.32446367e-02, -1.00246817e-01, -3.69973630e-02,\n",
       "         -6.39737248e-02, -2.99509317e-02, -2.28592813e-01,\n",
       "          4.42951918e-04,  1.00227281e-01,  8.44356418e-03,\n",
       "          1.24932930e-01,  3.99864763e-02,  5.82019836e-02,\n",
       "          7.17784017e-02, -1.31688401e-01, -2.40868181e-02,\n",
       "          7.65577108e-02, -3.61921638e-02,  1.35752112e-02,\n",
       "          2.93320566e-02, -1.78893611e-01,  8.25278908e-02,\n",
       "          9.08565074e-02,  1.27726793e-03, -6.44164383e-02,\n",
       "         -2.10052580e-02,  5.67823648e-04, -8.24061483e-02,\n",
       "         -5.44720888e-02,  2.07234815e-01, -1.74278140e-01,\n",
       "         -1.18030101e-01,  1.79677323e-01,  7.38099664e-02,\n",
       "         -1.20028101e-01, -1.76710203e-01,  2.23142818e-01,\n",
       "         -1.88847080e-01, -1.88121796e-01,  1.72192648e-01,\n",
       "         -1.26432002e-01, -8.74622762e-02,  1.68372467e-01,\n",
       "          2.06601128e-01,  2.03985557e-01,  1.42507479e-01,\n",
       "         -1.03798345e-01, -2.37980813e-01, -1.13417953e-01,\n",
       "         -1.71524316e-01,  1.74131230e-01,  2.10873380e-01,\n",
       "         -9.42959040e-02,  1.21171787e-01, -6.38326108e-02,\n",
       "          2.19890580e-01, -2.02155218e-01, -1.29148856e-01,\n",
       "         -2.38922849e-01, -5.41859865e-02, -2.38500416e-01,\n",
       "         -3.70079726e-02,  1.79292515e-01, -1.55989975e-01,\n",
       "         -1.23938844e-01,  1.52370408e-01,  1.36662871e-02,\n",
       "          2.23649889e-02, -1.34896994e-01, -7.83464909e-02,\n",
       "          1.51136890e-01, -1.24228142e-01,  8.85918885e-02,\n",
       "          1.48399487e-01,  1.84455708e-01,  3.16442549e-03,\n",
       "         -2.03874409e-01,  1.04537472e-01, -6.27755672e-02,\n",
       "         -6.27686679e-02,  1.43718526e-01, -5.77731729e-02,\n",
       "          1.00497022e-01, -1.00591928e-02, -1.37127578e-01,\n",
       "          2.14196593e-02, -1.09055310e-01, -1.85478181e-01,\n",
       "         -2.38989308e-01,  1.15395650e-01, -1.22722790e-01,\n",
       "         -3.12999040e-02, -1.62576586e-01,  2.18365043e-02,\n",
       "         -5.73845357e-02,  1.94726363e-01, -1.65940374e-01,\n",
       "         -9.87790525e-03],\n",
       "        [ 2.31559828e-01,  1.25581339e-01,  2.22963795e-01,\n",
       "          2.17031226e-01, -1.72396988e-01,  3.46875638e-02,\n",
       "          2.33982638e-01,  1.94397226e-01,  1.78199038e-01,\n",
       "          2.10107669e-01, -7.48144686e-02,  1.76178962e-02,\n",
       "         -2.19665900e-01, -1.20660670e-01, -5.45161963e-02,\n",
       "         -4.60039079e-02,  1.91600755e-01, -2.20155522e-01,\n",
       "         -2.76754946e-02,  2.11809322e-01, -2.34083682e-02,\n",
       "          1.53914616e-01,  3.70755792e-03, -2.24102065e-01,\n",
       "          1.05840281e-01, -1.28837436e-01,  9.56774801e-02,\n",
       "         -5.42557389e-02, -6.88394904e-02,  2.35680923e-01,\n",
       "          3.99884731e-02, -2.12421015e-01,  8.55984539e-02,\n",
       "          6.93427026e-03,  1.70284018e-01, -2.47270167e-02,\n",
       "         -5.49834818e-02, -9.47964787e-02,  1.34354681e-02,\n",
       "         -8.87858868e-03,  1.04271218e-01,  1.63830653e-01,\n",
       "          8.48351419e-03,  1.38177112e-01,  8.34735781e-02,\n",
       "          2.34234110e-01, -1.95840180e-01, -1.14745826e-01,\n",
       "          1.43927589e-01, -1.04492396e-01,  2.24538729e-01,\n",
       "          2.21476898e-01,  1.47133216e-01, -2.15399519e-01,\n",
       "         -3.81327271e-02, -2.09834561e-01,  1.53986767e-01,\n",
       "         -1.21521950e-01,  5.75291365e-02, -1.79057971e-01,\n",
       "         -3.96638364e-02,  9.14708823e-02, -2.11124033e-01,\n",
       "          9.13354009e-02,  9.03025717e-02,  1.74713925e-01,\n",
       "          6.89338297e-02, -1.46549135e-01, -6.62915111e-02,\n",
       "         -1.14725880e-01, -8.43759924e-02,  1.14874586e-01,\n",
       "         -1.68003142e-03,  2.34949872e-01, -9.85706002e-02,\n",
       "          1.83694854e-01, -1.05209202e-01,  1.72214761e-01,\n",
       "         -1.21917762e-01,  2.33381256e-01, -4.30410057e-02,\n",
       "          1.90200731e-01, -1.57085210e-01, -9.46874917e-03,\n",
       "         -1.23809703e-01,  8.64032358e-02,  7.37745017e-02,\n",
       "          6.72219247e-02, -6.05955273e-02, -1.20737955e-01,\n",
       "         -2.07038879e-01,  1.94105014e-01,  4.92850989e-02,\n",
       "          1.55040875e-01,  8.66612345e-02, -1.80490613e-01,\n",
       "         -1.25825942e-01,  2.32283160e-01, -3.70401740e-02,\n",
       "          2.58339196e-02],\n",
       "        [ 1.08918548e-03,  1.74395844e-01,  1.34854659e-01,\n",
       "          2.45445967e-03,  2.94101685e-02, -4.65469360e-02,\n",
       "         -1.62639380e-01, -8.20839703e-02, -6.75114989e-03,\n",
       "         -5.08496165e-03, -1.89562857e-01,  7.21776336e-02,\n",
       "          6.88960403e-02,  1.91716388e-01, -1.54001713e-01,\n",
       "         -1.29029334e-01,  8.99455100e-02, -2.18898088e-01,\n",
       "          2.11271003e-01,  1.24586746e-01, -7.46946633e-02,\n",
       "          1.97400585e-01, -2.03721792e-01, -7.52463490e-02,\n",
       "          2.15383902e-01, -3.76242846e-02,  1.98230907e-01,\n",
       "          5.49098402e-02,  1.46697387e-01, -1.41894221e-02,\n",
       "         -6.16732091e-02,  1.89414769e-02,  5.30540943e-03,\n",
       "         -9.26639736e-02,  1.82842806e-01,  2.13772506e-02,\n",
       "          1.56718045e-02,  9.89656299e-02,  1.27912268e-01,\n",
       "         -1.21878378e-01, -1.44205600e-01,  4.23761457e-02,\n",
       "          2.27419153e-01,  3.48234326e-02, -7.57579207e-02,\n",
       "          1.84258819e-03, -1.26370490e-02,  2.01434776e-01,\n",
       "         -5.95867485e-02,  1.34579375e-01,  1.37046203e-01,\n",
       "          2.25254461e-01, -2.34019235e-01,  5.24760932e-02,\n",
       "         -6.68910742e-02,  6.06523007e-02,  1.77527264e-01,\n",
       "         -1.89356089e-01,  6.85351640e-02,  1.16359740e-02,\n",
       "         -1.84268549e-01, -1.14853546e-01, -1.09491929e-01,\n",
       "         -1.54294372e-01, -1.50248095e-01,  9.91718918e-02,\n",
       "          1.72944978e-01,  4.33772653e-02,  3.51915509e-02,\n",
       "         -2.77453661e-03,  9.13001150e-02,  2.06179306e-01,\n",
       "          1.79651096e-01,  4.85248864e-03,  1.62395686e-02,\n",
       "         -1.42110884e-01,  2.00509593e-01, -6.73323125e-02,\n",
       "          2.23537996e-01, -9.92267132e-02,  9.35830325e-02,\n",
       "         -7.07975477e-02, -1.77601129e-01,  9.95092243e-02,\n",
       "         -4.58581746e-03,  2.31868789e-01, -1.91380590e-01,\n",
       "          1.78411797e-01,  1.35770887e-02, -1.87850505e-01,\n",
       "          2.22793326e-01,  2.08753452e-01,  1.48540810e-01,\n",
       "         -1.92071512e-01,  6.97945803e-02,  4.90484685e-02,\n",
       "         -2.41919607e-02,  6.41429871e-02, -2.38271818e-01,\n",
       "          2.23818347e-01],\n",
       "        [-1.18925862e-01, -1.03055432e-01,  6.09238595e-02,\n",
       "          9.86698717e-02,  1.95268020e-01,  1.58096895e-01,\n",
       "          2.01534286e-01,  2.30786070e-01, -1.47049084e-01,\n",
       "          9.33067948e-02,  1.09201446e-01,  2.23748073e-01,\n",
       "         -1.53872624e-01,  2.24177524e-01,  1.74428448e-01,\n",
       "          1.98026136e-01,  3.83766443e-02, -1.91938102e-01,\n",
       "          1.63912773e-04,  1.55044690e-01, -1.08051494e-01,\n",
       "         -3.38224620e-02, -1.24203466e-01, -2.19049871e-01,\n",
       "         -2.36113042e-01, -3.50039750e-02,  1.97176442e-01,\n",
       "          1.25542983e-01,  1.26846120e-01, -2.15057001e-01,\n",
       "         -6.90477490e-02, -5.42208105e-02, -2.10341513e-02,\n",
       "          4.08322066e-02, -2.18998849e-01, -2.20368221e-01,\n",
       "          1.14318281e-02, -1.94615617e-01,  1.89554080e-01,\n",
       "         -7.69449174e-02, -2.24787399e-01, -2.11704791e-01,\n",
       "         -5.24287224e-02, -2.17574596e-01, -1.67801917e-01,\n",
       "         -5.29556721e-02,  2.82726735e-02, -1.10691786e-03,\n",
       "          2.14643225e-01,  1.16818890e-01,  1.04618177e-01,\n",
       "         -5.15189469e-02, -5.61026037e-02, -4.29201275e-02,\n",
       "          4.29559201e-02,  2.38590464e-01,  1.90658078e-01,\n",
       "          1.76251069e-01,  1.62770823e-01,  1.32386521e-01,\n",
       "          9.97767597e-02, -2.36594513e-01, -1.33348152e-01,\n",
       "         -1.88092664e-01,  7.99342543e-02, -1.66266367e-01,\n",
       "          8.00358206e-02, -2.22458676e-01, -1.68635622e-01,\n",
       "         -2.10156634e-01,  4.10593003e-02, -3.21179777e-02,\n",
       "         -4.44651544e-02, -1.65488124e-02, -1.66313380e-01,\n",
       "          1.59226820e-01,  1.04822561e-01, -2.25434035e-01,\n",
       "         -1.22209735e-01, -2.34085396e-01, -2.12057114e-01,\n",
       "         -2.54243165e-02,  1.35741279e-01,  4.69673127e-02,\n",
       "         -4.07195836e-02,  1.32223517e-02, -1.09946564e-01,\n",
       "          8.35803896e-02, -1.69095218e-01, -4.51710075e-02,\n",
       "          6.45708889e-02,  1.27900094e-02,  5.34678251e-02,\n",
       "         -2.03038156e-01, -9.25744921e-02, -9.23191607e-02,\n",
       "          9.01709944e-02,  2.35219494e-01,  1.17570922e-01,\n",
       "         -1.78499788e-01]], dtype=float32)>,\n",
       " <tf.Variable 'dense_3/bias:0' shape=(100,) dtype=float32, numpy=\n",
       " array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       dtype=float32)>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取layer的变量\n",
    "layer.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense_3/kernel:0' shape=(5, 100) dtype=float32, numpy=\n",
       " array([[-1.87438279e-01,  1.17057130e-01, -1.99815318e-01,\n",
       "          2.22023234e-01, -1.62328482e-02, -2.33810067e-01,\n",
       "          3.29867154e-02, -1.45707473e-01, -2.31261343e-01,\n",
       "          8.25258344e-02, -3.88459265e-02, -1.85060874e-01,\n",
       "          1.59038469e-01,  2.17499152e-01,  1.54771969e-01,\n",
       "         -1.90375462e-01, -1.96589112e-01, -6.94796443e-04,\n",
       "         -1.38469934e-01, -7.61686116e-02,  1.67824551e-01,\n",
       "         -1.56438157e-01, -1.17947161e-03,  1.35876879e-01,\n",
       "         -1.52204037e-01,  2.02305511e-01,  1.01615176e-01,\n",
       "         -2.21040636e-01,  1.10610083e-01,  1.40476078e-02,\n",
       "         -2.17021137e-01,  2.17639163e-01,  3.83620709e-02,\n",
       "         -7.87134618e-02,  1.23965189e-01, -4.38629091e-02,\n",
       "          1.82106540e-01, -1.87780917e-01, -1.63990647e-01,\n",
       "         -2.54983008e-02, -3.36732566e-02, -1.94116876e-01,\n",
       "         -1.73759341e-01, -1.16186887e-02,  2.13929161e-01,\n",
       "          7.93157667e-02, -1.02152780e-01,  4.30196971e-02,\n",
       "         -1.16929226e-01, -2.57502645e-02, -6.97175860e-02,\n",
       "         -1.77903354e-01,  1.74064323e-01,  1.46787718e-01,\n",
       "         -1.80177033e-01,  9.60052460e-02,  1.66205093e-01,\n",
       "         -1.33879721e-01,  2.24551573e-01, -1.96295545e-01,\n",
       "          9.07173306e-02,  1.78772882e-01,  1.42574117e-01,\n",
       "          9.05241221e-02,  6.89752549e-02, -7.67410547e-02,\n",
       "          1.33813903e-01,  2.33949408e-01, -2.13704497e-01,\n",
       "         -1.65827915e-01,  1.93867818e-01,  7.07826763e-02,\n",
       "         -6.38635606e-02,  1.03244439e-01,  2.27196559e-01,\n",
       "         -1.42239273e-01,  2.61351317e-02, -1.19930021e-01,\n",
       "          6.50626272e-02,  5.25554270e-02, -2.20233828e-01,\n",
       "         -4.81836498e-02, -2.09291309e-01, -1.03912726e-01,\n",
       "          1.46329656e-01,  4.45419699e-02,  8.18864852e-02,\n",
       "         -2.36938074e-01, -9.47593153e-02,  3.00259441e-02,\n",
       "         -9.03265178e-02, -9.28232074e-03,  2.26433575e-03,\n",
       "         -1.97307348e-01,  3.36386710e-02,  4.93441969e-02,\n",
       "         -4.51138467e-02, -7.86980987e-03, -2.14031413e-01,\n",
       "          6.39158487e-03],\n",
       "        [ 1.91099659e-01,  1.19183213e-02, -1.48381293e-01,\n",
       "         -8.32446367e-02, -1.00246817e-01, -3.69973630e-02,\n",
       "         -6.39737248e-02, -2.99509317e-02, -2.28592813e-01,\n",
       "          4.42951918e-04,  1.00227281e-01,  8.44356418e-03,\n",
       "          1.24932930e-01,  3.99864763e-02,  5.82019836e-02,\n",
       "          7.17784017e-02, -1.31688401e-01, -2.40868181e-02,\n",
       "          7.65577108e-02, -3.61921638e-02,  1.35752112e-02,\n",
       "          2.93320566e-02, -1.78893611e-01,  8.25278908e-02,\n",
       "          9.08565074e-02,  1.27726793e-03, -6.44164383e-02,\n",
       "         -2.10052580e-02,  5.67823648e-04, -8.24061483e-02,\n",
       "         -5.44720888e-02,  2.07234815e-01, -1.74278140e-01,\n",
       "         -1.18030101e-01,  1.79677323e-01,  7.38099664e-02,\n",
       "         -1.20028101e-01, -1.76710203e-01,  2.23142818e-01,\n",
       "         -1.88847080e-01, -1.88121796e-01,  1.72192648e-01,\n",
       "         -1.26432002e-01, -8.74622762e-02,  1.68372467e-01,\n",
       "          2.06601128e-01,  2.03985557e-01,  1.42507479e-01,\n",
       "         -1.03798345e-01, -2.37980813e-01, -1.13417953e-01,\n",
       "         -1.71524316e-01,  1.74131230e-01,  2.10873380e-01,\n",
       "         -9.42959040e-02,  1.21171787e-01, -6.38326108e-02,\n",
       "          2.19890580e-01, -2.02155218e-01, -1.29148856e-01,\n",
       "         -2.38922849e-01, -5.41859865e-02, -2.38500416e-01,\n",
       "         -3.70079726e-02,  1.79292515e-01, -1.55989975e-01,\n",
       "         -1.23938844e-01,  1.52370408e-01,  1.36662871e-02,\n",
       "          2.23649889e-02, -1.34896994e-01, -7.83464909e-02,\n",
       "          1.51136890e-01, -1.24228142e-01,  8.85918885e-02,\n",
       "          1.48399487e-01,  1.84455708e-01,  3.16442549e-03,\n",
       "         -2.03874409e-01,  1.04537472e-01, -6.27755672e-02,\n",
       "         -6.27686679e-02,  1.43718526e-01, -5.77731729e-02,\n",
       "          1.00497022e-01, -1.00591928e-02, -1.37127578e-01,\n",
       "          2.14196593e-02, -1.09055310e-01, -1.85478181e-01,\n",
       "         -2.38989308e-01,  1.15395650e-01, -1.22722790e-01,\n",
       "         -3.12999040e-02, -1.62576586e-01,  2.18365043e-02,\n",
       "         -5.73845357e-02,  1.94726363e-01, -1.65940374e-01,\n",
       "         -9.87790525e-03],\n",
       "        [ 2.31559828e-01,  1.25581339e-01,  2.22963795e-01,\n",
       "          2.17031226e-01, -1.72396988e-01,  3.46875638e-02,\n",
       "          2.33982638e-01,  1.94397226e-01,  1.78199038e-01,\n",
       "          2.10107669e-01, -7.48144686e-02,  1.76178962e-02,\n",
       "         -2.19665900e-01, -1.20660670e-01, -5.45161963e-02,\n",
       "         -4.60039079e-02,  1.91600755e-01, -2.20155522e-01,\n",
       "         -2.76754946e-02,  2.11809322e-01, -2.34083682e-02,\n",
       "          1.53914616e-01,  3.70755792e-03, -2.24102065e-01,\n",
       "          1.05840281e-01, -1.28837436e-01,  9.56774801e-02,\n",
       "         -5.42557389e-02, -6.88394904e-02,  2.35680923e-01,\n",
       "          3.99884731e-02, -2.12421015e-01,  8.55984539e-02,\n",
       "          6.93427026e-03,  1.70284018e-01, -2.47270167e-02,\n",
       "         -5.49834818e-02, -9.47964787e-02,  1.34354681e-02,\n",
       "         -8.87858868e-03,  1.04271218e-01,  1.63830653e-01,\n",
       "          8.48351419e-03,  1.38177112e-01,  8.34735781e-02,\n",
       "          2.34234110e-01, -1.95840180e-01, -1.14745826e-01,\n",
       "          1.43927589e-01, -1.04492396e-01,  2.24538729e-01,\n",
       "          2.21476898e-01,  1.47133216e-01, -2.15399519e-01,\n",
       "         -3.81327271e-02, -2.09834561e-01,  1.53986767e-01,\n",
       "         -1.21521950e-01,  5.75291365e-02, -1.79057971e-01,\n",
       "         -3.96638364e-02,  9.14708823e-02, -2.11124033e-01,\n",
       "          9.13354009e-02,  9.03025717e-02,  1.74713925e-01,\n",
       "          6.89338297e-02, -1.46549135e-01, -6.62915111e-02,\n",
       "         -1.14725880e-01, -8.43759924e-02,  1.14874586e-01,\n",
       "         -1.68003142e-03,  2.34949872e-01, -9.85706002e-02,\n",
       "          1.83694854e-01, -1.05209202e-01,  1.72214761e-01,\n",
       "         -1.21917762e-01,  2.33381256e-01, -4.30410057e-02,\n",
       "          1.90200731e-01, -1.57085210e-01, -9.46874917e-03,\n",
       "         -1.23809703e-01,  8.64032358e-02,  7.37745017e-02,\n",
       "          6.72219247e-02, -6.05955273e-02, -1.20737955e-01,\n",
       "         -2.07038879e-01,  1.94105014e-01,  4.92850989e-02,\n",
       "          1.55040875e-01,  8.66612345e-02, -1.80490613e-01,\n",
       "         -1.25825942e-01,  2.32283160e-01, -3.70401740e-02,\n",
       "          2.58339196e-02],\n",
       "        [ 1.08918548e-03,  1.74395844e-01,  1.34854659e-01,\n",
       "          2.45445967e-03,  2.94101685e-02, -4.65469360e-02,\n",
       "         -1.62639380e-01, -8.20839703e-02, -6.75114989e-03,\n",
       "         -5.08496165e-03, -1.89562857e-01,  7.21776336e-02,\n",
       "          6.88960403e-02,  1.91716388e-01, -1.54001713e-01,\n",
       "         -1.29029334e-01,  8.99455100e-02, -2.18898088e-01,\n",
       "          2.11271003e-01,  1.24586746e-01, -7.46946633e-02,\n",
       "          1.97400585e-01, -2.03721792e-01, -7.52463490e-02,\n",
       "          2.15383902e-01, -3.76242846e-02,  1.98230907e-01,\n",
       "          5.49098402e-02,  1.46697387e-01, -1.41894221e-02,\n",
       "         -6.16732091e-02,  1.89414769e-02,  5.30540943e-03,\n",
       "         -9.26639736e-02,  1.82842806e-01,  2.13772506e-02,\n",
       "          1.56718045e-02,  9.89656299e-02,  1.27912268e-01,\n",
       "         -1.21878378e-01, -1.44205600e-01,  4.23761457e-02,\n",
       "          2.27419153e-01,  3.48234326e-02, -7.57579207e-02,\n",
       "          1.84258819e-03, -1.26370490e-02,  2.01434776e-01,\n",
       "         -5.95867485e-02,  1.34579375e-01,  1.37046203e-01,\n",
       "          2.25254461e-01, -2.34019235e-01,  5.24760932e-02,\n",
       "         -6.68910742e-02,  6.06523007e-02,  1.77527264e-01,\n",
       "         -1.89356089e-01,  6.85351640e-02,  1.16359740e-02,\n",
       "         -1.84268549e-01, -1.14853546e-01, -1.09491929e-01,\n",
       "         -1.54294372e-01, -1.50248095e-01,  9.91718918e-02,\n",
       "          1.72944978e-01,  4.33772653e-02,  3.51915509e-02,\n",
       "         -2.77453661e-03,  9.13001150e-02,  2.06179306e-01,\n",
       "          1.79651096e-01,  4.85248864e-03,  1.62395686e-02,\n",
       "         -1.42110884e-01,  2.00509593e-01, -6.73323125e-02,\n",
       "          2.23537996e-01, -9.92267132e-02,  9.35830325e-02,\n",
       "         -7.07975477e-02, -1.77601129e-01,  9.95092243e-02,\n",
       "         -4.58581746e-03,  2.31868789e-01, -1.91380590e-01,\n",
       "          1.78411797e-01,  1.35770887e-02, -1.87850505e-01,\n",
       "          2.22793326e-01,  2.08753452e-01,  1.48540810e-01,\n",
       "         -1.92071512e-01,  6.97945803e-02,  4.90484685e-02,\n",
       "         -2.41919607e-02,  6.41429871e-02, -2.38271818e-01,\n",
       "          2.23818347e-01],\n",
       "        [-1.18925862e-01, -1.03055432e-01,  6.09238595e-02,\n",
       "          9.86698717e-02,  1.95268020e-01,  1.58096895e-01,\n",
       "          2.01534286e-01,  2.30786070e-01, -1.47049084e-01,\n",
       "          9.33067948e-02,  1.09201446e-01,  2.23748073e-01,\n",
       "         -1.53872624e-01,  2.24177524e-01,  1.74428448e-01,\n",
       "          1.98026136e-01,  3.83766443e-02, -1.91938102e-01,\n",
       "          1.63912773e-04,  1.55044690e-01, -1.08051494e-01,\n",
       "         -3.38224620e-02, -1.24203466e-01, -2.19049871e-01,\n",
       "         -2.36113042e-01, -3.50039750e-02,  1.97176442e-01,\n",
       "          1.25542983e-01,  1.26846120e-01, -2.15057001e-01,\n",
       "         -6.90477490e-02, -5.42208105e-02, -2.10341513e-02,\n",
       "          4.08322066e-02, -2.18998849e-01, -2.20368221e-01,\n",
       "          1.14318281e-02, -1.94615617e-01,  1.89554080e-01,\n",
       "         -7.69449174e-02, -2.24787399e-01, -2.11704791e-01,\n",
       "         -5.24287224e-02, -2.17574596e-01, -1.67801917e-01,\n",
       "         -5.29556721e-02,  2.82726735e-02, -1.10691786e-03,\n",
       "          2.14643225e-01,  1.16818890e-01,  1.04618177e-01,\n",
       "         -5.15189469e-02, -5.61026037e-02, -4.29201275e-02,\n",
       "          4.29559201e-02,  2.38590464e-01,  1.90658078e-01,\n",
       "          1.76251069e-01,  1.62770823e-01,  1.32386521e-01,\n",
       "          9.97767597e-02, -2.36594513e-01, -1.33348152e-01,\n",
       "         -1.88092664e-01,  7.99342543e-02, -1.66266367e-01,\n",
       "          8.00358206e-02, -2.22458676e-01, -1.68635622e-01,\n",
       "         -2.10156634e-01,  4.10593003e-02, -3.21179777e-02,\n",
       "         -4.44651544e-02, -1.65488124e-02, -1.66313380e-01,\n",
       "          1.59226820e-01,  1.04822561e-01, -2.25434035e-01,\n",
       "         -1.22209735e-01, -2.34085396e-01, -2.12057114e-01,\n",
       "         -2.54243165e-02,  1.35741279e-01,  4.69673127e-02,\n",
       "         -4.07195836e-02,  1.32223517e-02, -1.09946564e-01,\n",
       "          8.35803896e-02, -1.69095218e-01, -4.51710075e-02,\n",
       "          6.45708889e-02,  1.27900094e-02,  5.34678251e-02,\n",
       "         -2.03038156e-01, -9.25744921e-02, -9.23191607e-02,\n",
       "          9.01709944e-02,  2.35219494e-01,  1.17570922e-01,\n",
       "         -1.78499788e-01]], dtype=float32)>,\n",
       " <tf.Variable 'dense_3/bias:0' shape=(100,) dtype=float32, numpy=\n",
       " array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       dtype=float32)>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取layer的可训练变量\n",
    "layer.trainable_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 查看layer的帮助\n",
    "help(layer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用子类继承的方式自定义dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# customized dense layer\n",
    "# 使用子类继承的方式自定义dense layer\n",
    "class CustomizedDenseLayer(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        \"\"\"初始化\"\"\"\n",
    "        self.units = units\n",
    "        self.activation = keras.layers.Activation(activation)\n",
    "        super(CustomizedDenseLayer, self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        \"\"\"\n",
    "        构建所需要的参数 x * w + b\n",
    "        input_shape:[None,a]\n",
    "        w:[a,b]\n",
    "        output_shape:[None,b]\n",
    "        \"\"\"\n",
    "        self.kernel = self.add_weight(name='kernel',\n",
    "                                      shape=(input_shape[1],self.units),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.bias = self.add_weight(name='bias',\n",
    "                                      shape=(self.units, ),\n",
    "                                      initializer='zeros',\n",
    "                                      trainable=True)\n",
    "        super(CustomizedDenseLayer, self).build(input_shape)\n",
    "    \n",
    "    def call(self, x):\n",
    "        \"\"\"完整的正向计算\"\"\"\n",
    "        return self.activation(x @ self.kernel + self.bias)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "customized_dense_layer (Cust (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "customized_dense_layer_1 (Cu (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 构建模型\n",
    "model = keras.models.Sequential([\n",
    "    CustomizedDenseLayer(30, activation='relu',input_shape=x_train.shape[1:]),\n",
    "    CustomizedDenseLayer(1)\n",
    "])\n",
    "model.summary()\n",
    "\n",
    "# 模型编译，固化模型\n",
    "model.compile(loss='mean_squared_error',optimizer='adam')\n",
    "\n",
    "# 回调函数\n",
    "callbacks = [keras.callbacks.EarlyStopping(patience=5,min_delta=1e-2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/100\n",
      "11610/11610 [==============================] - 2s 157us/sample - loss: 2.0159 - val_loss: 0.6545\n",
      "Epoch 2/100\n",
      "11610/11610 [==============================] - 1s 118us/sample - loss: 0.5035 - val_loss: 0.4744\n",
      "Epoch 3/100\n",
      "11610/11610 [==============================] - 1s 118us/sample - loss: 0.4369 - val_loss: 0.4482\n",
      "Epoch 4/100\n",
      "11610/11610 [==============================] - 1s 118us/sample - loss: 0.4155 - val_loss: 0.4305\n",
      "Epoch 5/100\n",
      "11610/11610 [==============================] - 1s 118us/sample - loss: 0.4073 - val_loss: 0.4184\n",
      "Epoch 6/100\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.4006 - val_loss: 0.4114\n",
      "Epoch 7/100\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.3955 - val_loss: 0.4077\n",
      "Epoch 8/100\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.3905 - val_loss: 0.4013\n",
      "Epoch 9/100\n",
      "11610/11610 [==============================] - 1s 119us/sample - loss: 0.3865 - val_loss: 0.3994\n",
      "Epoch 10/100\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.3815 - val_loss: 0.3921\n",
      "Epoch 11/100\n",
      "11610/11610 [==============================] - 1s 119us/sample - loss: 0.3772 - val_loss: 0.3868\n",
      "Epoch 12/100\n",
      "11610/11610 [==============================] - 1s 122us/sample - loss: 0.3741 - val_loss: 0.3808\n",
      "Epoch 13/100\n",
      "11610/11610 [==============================] - 1s 119us/sample - loss: 0.3702 - val_loss: 0.3776\n",
      "Epoch 14/100\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.3653 - val_loss: 0.3796\n",
      "Epoch 15/100\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.3634 - val_loss: 0.3684\n",
      "Epoch 16/100\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.3604 - val_loss: 0.3629\n",
      "Epoch 17/100\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.3569 - val_loss: 0.3707\n",
      "Epoch 18/100\n",
      "11610/11610 [==============================] - 1s 119us/sample - loss: 0.3540 - val_loss: 0.3563\n",
      "Epoch 19/100\n",
      "11610/11610 [==============================] - 1s 111us/sample - loss: 0.3517 - val_loss: 0.3528\n",
      "Epoch 20/100\n",
      "11610/11610 [==============================] - 1s 116us/sample - loss: 0.3489 - val_loss: 0.3488\n",
      "Epoch 21/100\n",
      "11610/11610 [==============================] - 1s 112us/sample - loss: 0.3462 - val_loss: 0.3454\n",
      "Epoch 22/100\n",
      "11610/11610 [==============================] - 1s 112us/sample - loss: 0.3427 - val_loss: 0.3472\n",
      "Epoch 23/100\n",
      "11610/11610 [==============================] - 1s 111us/sample - loss: 0.3417 - val_loss: 0.3423\n",
      "Epoch 24/100\n",
      "11610/11610 [==============================] - 1s 111us/sample - loss: 0.3407 - val_loss: 0.3438\n",
      "Epoch 25/100\n",
      "11610/11610 [==============================] - 1s 108us/sample - loss: 0.3391 - val_loss: 0.3374\n",
      "Epoch 26/100\n",
      "11610/11610 [==============================] - 1s 107us/sample - loss: 0.3367 - val_loss: 0.3371\n"
     ]
    }
   ],
   "source": [
    "history_cus_subclass = model.fit(x_train_scaled,y_train,epochs=100,\n",
    "                                 validation_data=(x_valid_scaled,y_valid),\n",
    "                                 callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAEzCAYAAAD+XEDdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZxddX3/8dfnnrvPnsnMZGMJARJDhgCySNUwooCBKuJSQWsBF0rdrbWI/f1caqtVam37K4rUWqR1gSJVqnGtjMADRAQTkrCEEIRM9m22O3P37++Pc2fNhNybzOTeO/f9fDzO45x77jl3vvPlkvd8v+d7vsecc4iIiEhlCpS7ACIiInJoCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQp22KA2s2+Y2W4z23CI983M/tnMNpvZ42Z21vQXU0REpDYV06K+DXjti7y/GjilsFwHfPXoiyUiIiJQRFA75+4D9r/IIZcDtzvfr4FmM5s/XQUUERGpZdNxjXohsHXc657CPhERETlKwWn4DJti35TzkprZdfjd40Sj0Zcef/zxRf2AA0lHf9pxQqPGvpUin88TCKjOZprqeeapjmee6vjY2LRp017nXFsp50xHUPcAx417vQjYPtWBzrlbgVsBli5d6p5++umifsA//eIZvvyLTTzxuUvxAlP9XSBT6e7upqurq9zFmPVUzzNPdTzzVMfHhpk9X+o50/Hn0z3AnxRGf78M6HPO7ZiGzx0VD3sADKWz0/mxIiIiFe+wLWoz+w7QBcw1sx7gU0AIwDl3C7AGuBTYDAwB1053IWOFoB5O52iIhqb740VERCrWYYPaOXfVYd53wPumrURTGGtR52byx4iIiFSc6bhGPeMU1CIi5ZfJZOjp6SGZTJa7KBUvGo2yaNEiQqGj7wWuiqCOhf1iDmd0jVpEpFx6enpoaGjgxBNPxEwDew/FOce+ffvo6elh8eLFR/15VTEWXy1qEZHySyaTtLa2KqQPw8xobW2dtp4HBbWIiBRNIV2c6aynKglqv+tbt2eJiNS2+vr6chfhmKuSoFaLWkREalNVBPX4+6hFREScc3zsYx9jxYoVdHZ2cscddwCwY8cOVq1axRlnnMGKFSu4//77yeVyXHPNNaPHfvnLXy5z6UtTFaO+4yG1qEVEZMzdd9/N2rVrWbduHXv37uWcc85h1apVfPvb3+aSSy7hr/7qr8jlcgwNDbF27Vq2bdvGhg0bAOjt7S1z6UtTFUEd9AKEvYCCWkSkQnzmfzbyxPb+af3M5Qsa+dTrTivq2AceeICrrroKz/Po6Ojgggsu4JFHHuGcc87hne98J5lMhje84Q2cccYZnHTSSWzZsoUPfOADXHbZZVx88cXTWu6ZVhVd3+B3fw9rMJmIiOB3fU9l1apV3HfffSxcuJB3vOMd3H777bS0tLBu3Tq6urq4+eabefe7332MS3t0qqJFDf6AMrWoRUQqQ7Et35myatUqvva1r3H11Vezf/9+7rvvPm666Saef/55Fi5cyHve8x4SiQSPPfYYl156KeFwmDe96U0sWbKEa665pqxlL1V1BXVGQS0iInDFFVfw0EMPsXLlSsyML37xi8ybN49vfvOb3HTTTYRCIerr67n99tvZtm0b1157Lfl8HoDPf/7zZS59aaooqIMMpdT1LSJSywYHBwF/QpGbbrqJm266acL7V199NVdfffVB5z322GPHpHwzoaquUavrW0REak3VBHU87DGsrm8REakxVRXUalGLiEitqZqgjoWCmplMRERqTtUEtd+i1mAyERGpLVUW1GpRi4hIbamioA6SyubJ5aeejUZERGQ2qqKgHnkwh7q/RUSkOC/2/Orf//73rFix4hiW5shUTVDrUZciIlKLqiaox1rUCmoRkVp1ww038JWvfGX09ac//Wk+85nP8OpXv5qzzjqLzs5OfvCDH5T8uclkkmuvvZbOzk7OPPNM7r33XgA2btzIueeeyxlnnMHpp5/OM888QyKR4LLLLmPlypWsWLFi9FnYM6WKphBVUIuIVIwffxx2rp/ez5zXCav/7kUPufLKK/nwhz/Me9/7XgDuvPNOfvKTn/CRj3yExsZG9u7dy8te9jJe//rXY2ZF/+ibb74ZgPXr1/PUU09x8cUXs2nTJm655RY+9KEP8fa3v510Ok0ul2PNmjUsWLCAH/3oRwD09fUd4S9cnKppUcfC/t8UwxldoxYRqVVnnnkmu3fvZvv27axbt46Wlhbmz5/PJz7xCU4//XRe85rXsG3bNnbt2lXS5z7wwAO84x3vAGDZsmWccMIJbNq0ifPPP5/Pfe5zfOELX+D5558nFovR2dnJL37xC2644Qbuv/9+mpqaZuJXHaUWtYiIlO4wLd+Z9OY3v5m77rqLnTt3cuWVV/Ktb32LPXv28OijjxIKhTjxxBNJJpMlfeahnm/9tre9jfPOO48f/ehHXHLJJXz961/nwgsv5NFHH2XNmjXceOONXHzxxXzyk5+cjl9tSlUT1LGQglpERPzu7/e85z3s3buXX/3qV9x55520t7cTCoW49957ef7550v+zFWrVvGtb32LCy+8kE2bNvHCCy+wdOlStmzZwkknncQHP/hBtmzZwuOPP86yZcuYM2cOf/zHf0x9fT233Xbb9P+S41RNUMc16ltERIDTTjuNgYEBFi5cyPz583n729/O6173Os4++2zOOOMMli1bVvJnvve97+X666+ns7OTYDDIbbfdRiQS4Y477uA///M/CYVCzJs3j09+8pM88sgjfOxjHyMQCBAKhfjqV786A7/lmKoJ6rqIX1S1qEVEZP36sYFsc+fO5aGHHpryuJHnV0/lxBNPZMOGDQBEo9EpW8Y33ngjN95444R9l1xyCZdccskRlPrIVNFgMk14IiIitadqWtRxXaMWEZEjsH79+tER3SMikQgPP/xwmUpUmqoJ6qAXIOwFFNQiIlKSzs5O1q5dW+5iHLGq6foGv/t7WF3fIiJlc6jbmGSi6aynqgpqPepSRKR8otEo+/btU1gfhnOOffv2EY1Gp+XzqqbrG/wW9VBGQS0iUg6LFi2ip6eHPXv2lLsoFS8ajbJo0aJp+ayqCup42NN91CIiZRIKhVi8eHG5i1FzqqzrO6jbs0REpKZUWVDrGrWIiNQWBbWIiEgFq6qgjoWCukYtIiI1paqC2m9R6xq1iIjUjioMarWoRUSkdlRVUMfCHqlsnlxeN9uLiEhtqKqgHn0mtSY9ERGRGlFlQT3yTGpdpxYRkdpQZUFdeNRlSi1qERGpDUUFtZm91syeNrPNZvbxKd5vMrP/MbN1ZrbRzK6d/qKOC2oNKBMRkRpx2KA2Mw+4GVgNLAeuMrPlkw57H/CEc24l0AV8yczC01xWYoWu7+GMur5FRKQ2FNOiPhfY7Jzb4pxLA98FLp90jAMazMyAemA/MO1pqha1iIjUmmKenrUQ2DrudQ9w3qRj/gW4B9gONABvdc7lJ3+QmV0HXAfQ1tZGd3d3SYV9vt8P6N88to7ctqp68FdZDA4OllzHUjrV88xTHc881XHlKibtbIp9k29kvgRYC1wILAF+bmb3O+f6J5zk3K3ArQBLly51XV1dJRV2y55BePBXLDn1JXSdubCkc2tRd3c3pdaxlE71PPNUxzNPdVy5iun67gGOG/d6EX7LebxrgbudbzPwHLBseoo4Zuz2LHV9i4hIbSgmqB8BTjGzxYUBYlfid3OP9wLwagAz6wCWAlums6AA8cjINWoNJhMRkdpw2K5v51zWzN4P/BTwgG845zaa2fWF928BPgvcZmbr8bvKb3DO7Z3uwsZDhZnJ1KIWEZEaUdSILOfcGmDNpH23jNveDlw8vUU7WNALEPYCJBTUIiJSI6pqZjLwH8wxrK5vERGpEVUX1HrUpYiI1JKqC+pY2GNIT88SEZEaUXVBHQ97GkwmIiI1o/qCOhTU7VkiIlIzqi6oY2pRi4hIDam6oK6LaDCZiIjUjqoL6lgoqKAWEZGaUXVB7d+epWvUIiJSG6o0qNWiFhGR2lB1QR0Le6SyeXL5yU/aFBERmX2qLqjj4cKDOTTpiYiI1ICqC+rY6DOpdZ1aRERmv6oLaj3qUkREaknVBXVdxA9qDSgTEZFaUHVBra5vERGpJVUX1CODydSiFhGRWlB1QR0LKahFRKR2VF1Qj96epaAWEZEaUIVBPXKNWkEtIiKzX9UFdWz0GrUGk4mIyOxXdUGtrm8REaklVRfUIS9A2AswpClERUSkBlRdUIPf/a0WtYiI1IKqDOp42COR0jVqERGZ/aoyqGNhT13fIiJSE6oyqOPq+hYRkRpRnUEdCur2LBERqQlVGdQaTCYiIrWiKoM6HvY0M5mIiNSEqgzqmIJaRERqRFUGdV04yLBGfYuISA2oyqDWfdQiIlIryhbUdUNbwbkjOjcW9khl8+TyR3a+iIhItShbUAdyKdj26BGdO/pgDnV/i4jILFe+rm8LwLrvHtGpsdFnUqv7W0REZreyBXXWi8OGuyCbLvnceEiPuhQRkdpQtqDOhBph+ABs/nnJ5450fesWLRERme3K16IOxqGuDdZ9p+RzYwpqERGpEeW9PWvFm2HTT/2WdQnqIv41anV9i4jIbFfeoF55JeTSsPG/SzotVrhGndBgMhERmeXKG9TzV0LbspJHf4/enqUWtYiIzHLlDWozv1W99WHYv6Xo0+Kjt2cpqEVEZHYr/xSinX8EGKy7o+hTxgaTqetbRERmt/IHddNCWPxKePyOoqcUVde3iIjUiqKC2sxea2ZPm9lmM/v4IY7pMrO1ZrbRzH5VUilWXgUHnoOtvynq8JAXIOQZQ5pCVEREZrnDBrWZecDNwGpgOXCVmS2fdEwz8BXg9c6504C3lFSKl7wOgrGS7qmOhTy1qEVEZNYrpkV9LrDZObfFOZcGvgtcPumYtwF3O+deAHDO7S6pFJEGP6w33g3ZVFGn1EWCukYtIiKzXjFBvRDYOu51T2HfeKcCLWbWbWaPmtmflFySlW+FZB9s+klRh8fCHgm1qEVEZJYLFnGMTbFv8qivIPBS4NVADHjIzH7tnNs04YPMrgOuA2hra6O7u3vsvbzxsnALA/97Mxt2Nx22ULnUMD07khM+QyYaHBxU/RwDqueZpzqeearjylVMUPcAx417vQjYPsUxe51zCSBhZvcBK4EJQe2cuxW4FWDp0qWuq6tr4qdk3k7k4a/RdU4n1LW+aKE6nnqIQAC6us4v4leoTd3d3RxUxzLtVM8zT3U881THlauYru9HgFPMbLGZhYErgXsmHfMD4JVmFjSzOHAe8GTJpVl5JeQz/rXqw4iFNZhMRERmv8MGtXMuC7wf+Cl++N7pnNtoZteb2fWFY54EfgI8DvwG+LpzbkPJpZnXCR0rippSNB72NDOZiIjMesV0feOcWwOsmbTvlkmvbwJuOuoSnf5W+Pn/hb3PwNxTDnlYTEEtIiI1oPwzk03W+RawgD9T2YuIhz2GNeGJiIjMcpUX1I3z4aQuP6jz+UMeVhfWfdQiIjL7VV5QA5x+JfS+AC88dMhDYmGPZCZPLl/c/OAiIiLVqDKD+iV/CKE6ePzQg8pGH8yh7m8REZnFKjOow3Ww/PWw8QeQGZ7ykNjoM6nV/S0iIrNXZQY1+PdUp/rg6R9P+XY8pEddiojI7Fe5QX3iK6FhwSFHf490fesWLRERmc0qN6gDHpz+Fnjm5zC456C3YwpqERGpAZUb1OCP/nY52PC9g96KF65Rq+tbRERms8oO6o7lMO/0KUd/j3V9azCZiIjMXpUd1AArr4Ltv4M9T0/YrduzRESkFlR+UHe+Gcw76EEdI13fiZSCWkREZq/KD+r6dlhyITx+54QpRWPq+hYRkRpQ+UEN/j3V/T3w/AOju0a7vjWYTEREZrHqCOpll0G4AdaN3VMd8gKEPGNI16hFRGQWq46gDsXgtMvhie9Demh0dyzkqUUtIiKzWnUENfj3VKcH4ek1o7vietSliIjMctUT1Ce8HJqOg3XfGd0VD3uamUxERGa16gnqQABO/yN49pcwsAuAeERd3yIiMrtVT1BDYUrRPGy4C4B4KEhCXd8iIjKLVVdQt50KC84a7f6OhT12D6TI5vKHOVFERKQ6VVdQg39P9c71sGsjl3bOY8ueBB+5c53CWkREZqXqC+oVb4JAENZ9l7eeczw3rl7G/6zbzl/81zpyeVfu0omIiEyr6gvqurlw8kWw/r8gn+NPL1jCxy5ZyvfXbudjdymsRURkdqm+oAZY+VYY2AHP3QfA+151Mh+96FTufmwbH//e4+QV1iIiMksEy12AI3Lqaog0+U/UWvIqAD7w6lPI5h3/9L/P4AWMz13RSSBgZS6oiIjI0anOoA5F4bQ3wPq7IPUliNQD8OHXnELeOf7fLzcTCBh/+4YVmCmsRUSkelVn1zfAyqsgk4Cnfji6y8z484tO5c+6lvDth1/gkz/YiHPqBhcRkepVnS1qgONfBs0nwL2fg1waTnsjROoxM/7ykqXk846v3bcFL2B86nXL1bIWEZGqVL0tajP4wy/7T9a65wPwpWXwwz+HHY9jZnx89TLe9YrF3Pbg7/mbHz2plrWIiFSl6m1RA5z8aljya9j6MDx6G6z9Fvz232DBWdhLr+H/XPRGcnnHvz3wHF7AuHH1MrWsRUSkqlRvi3qEmd8NfsUt8NGnYPUXITMM//NB7EvL+JT9K3+5MsWt923hiz99Wi1rERGpKtUf1OPFWuC8P4X3PgTv/Bm85PXYuu/w3qev5YGWv+bA/f/KP//4dwprERGpGrMrqEeYwfHnwRVfHW1lL2wI8Hehr/Ouh1ez/pZ3wva15S6liIjIYc3OoB6v0Mq2P3uQ/LU/48mWV3HKzh/CrRfA1y6A3/47pAbKXUoREZEpzf6gHmFG4ITzOOuD3+Gzy+7m/2auYW//IPzww/D3S+Gud8Hv/hP6tpW7pCIiIqOqe9T3EfACxmff+go+avWcvfYi/uEPMrzR/QI2/Qw23OUfNPdUOOlV/vSkJ74CIg3lLbSIiNSsmgtq8MP679+ykpyDP39wO/sv+yjv/ot/gV0bYcu98Oy98Njt8Juv+Y/UXHTOWHAvOAu8mqw2EREpg5pNnKAX4Mt/tJJ83vE3P3qSLXsTXHHmQl76svcT+IMPQCbp3589Etzdn4fuz/kPA1n8Sjipyw/v1iX+4DUREZEZULNBDX5Y/+OVZ9AYC3HXoz18++EXaG+I8NoV81i9Yj7nLl6Fd9IF8JpPw9B+2NJdWO4dm2O86Tg/tJe8ChZ3QV1rmX4bERGZjWo6qAFCXoDPv7GTv7rsJfzyqd38eP0O7vztVm5/6Hla68JcfNo8Vq+Yx/lLWgmteCOseCM4B/u3jLW2n7gHfvcfgPkt7I4VMG8FzDvd325coFa3iIgckZoP6hH1kSCvX7mA169cwFA6y6+e3sOaDTu5Z+02vvObF2iKhbh4eQerO+fx8pPnEmld4ofyOe+GXBa2/w6e6/bvz97+O3ji+2MfHmsphHenv3SsgLalEIyU7fcVEZHqoKCeQjwcZHXnfFZ3zieZyXH/M3v58fod/GTDTv7r0R4aIkFes7yD1SvmserUNqKhIBx3jr+MSPb7g9N2bYCd6/31b/8dssP++4EgzF1aaHl3jgV53dzy/NIiIlKRFNSHEQ15XLS8g4uWd5DK5nhw8z7WrN/Bz5/cxX//bhvxsMeFy9q5tHM+XUvbiIcLVRpthBPO95cR+RzsexZ2rYedG/zwfu5+ePyOsWPq5/nh3Xw8NCzwu80b549tRxuPbQWIiEhZKahLEAl6vGpZO69a1k4ml+fXW/axZv1OfrZxJz98fAfRUIBTOxpY1BJjUUu8sPa3FzbHqIsEoe1Uf1nxprEPTuybGN67NsK2x2B4/8GFCNdDw3w/vBsXFrYLIT6yXdcGAe/YVYyIiMyYooLazF4L/BPgAV93zv3dIY47B/g18Fbn3F3TVsoKFPICvPKUNl55Sht/84YV/Oa5/fzsiZ1s3j3IUzsG+MWTu0ln8xPOaYmHDgrw0SBf+ArqT+qa+EMySRjYAf3bx9b922FgO/Tv8Fvjgzshn514XiAI9fM4y8Vh63H+hC3RRog0+tuj6/H7x+0L12nwm4hIhThsUJuZB9wMXAT0AI+Y2T3OuSemOO4LwE9noqCVzAsY5y9p5fwlY7dm5fOOvYkUPQeGC8sQ2wrbm3YN8MundpOaFOTN8RCLWmIsbI4xvylGe2OEjoYoHY1L6Wg/nfaTozRGgxOfqZ3PQ2IP9G87KNSzLzwJyV7ofcGfzzw1AJnE4X8hC4wL9EaINkF9OzTM89f186Cho7CeB7E5EKid2WhFRI6lYlrU5wKbnXNbAMzsu8DlwBOTjvsA8D3gHIRAwGhviNLeEOWs41sOet85x97BNNt6/RAfCfOeA8Ns2ZPgwWf3MZDMHnReNBSgozFKR0PUD/LGKB2NEToaO2hvOIGOxf6+ukiQx7u76erqmvgBuSykC6Gd7B8L8FR/YZli/3Cv3x3/7C/91wf9sqFCgHcUwvwQ63greGG11kVESlBMUC8Eto573QOcN/4AM1sIXAFciIK6KGZGW0OEtoYIZxzXPOUxQ+ksu/tT7OpPsmsgxe7+pL9d2Ldxez//++RuhjO5g86tjwSp93Ic99SDtMTDzKkL01IXZk68sK6L0BJfyJz6MC0dYRoik1rqh5JOwOAuGNjld7tPXh94Hrb+Bob2HvozvAgEo/7taSOLFznE6ygEw/56ZF+0yb8+P3JtvnGBbnUTkVmrmKCe6l9vN+n1PwI3OOdyL/aPvZldB1wH0NbWRnd3d5HFlMbCcnIdUAfM9/c7FyGZgwNJR2+qsCTzHEg59iXyDPb3sXMfDKQdA2lHbvJ/uQLPoD5sNIT8dX3IaAjbxO0Q1IfG9sWCzZi1QHAZtOAvBZbPEk73Ek4fIJzeTzh9gFBmgEA+M25JY25sO5DMEsgPEsjvJ5DP+vsmHevl0hj5g8qfDjWRiswlFWk9aJ2MziUdbiXvhaf3P0rB4OCgvsszTHU881THlauYoO4Bjhv3ehGwfdIxZwPfLYT0XOBSM8s6574//iDn3K3ArQBLly51B3XLyrTqntT17Zwjkc5xIJFmXyLNgUSa/Yk0B4YmrRMZ9g+leW6/vy9/iHAPBozmeJiWeIiWusK60GJviYdoaQwXXoeoi4dpjodpiAYJeUd5PTvZX7gOv82/Jt+3jXD/NsL922no3w77n4Zk38HnxVsLLfBF/rq+w5/yNd4K8bn+PezxVv+aewkPXplczzL9VMczT3VcuYr51+gR4BQzWwxsA64E3jb+AOfc4pFtM7sN+OHkkJbyMzO/SzwS5Lg58aLOyecd/ckMB4Yy7E+k6S2Eee9QhgNDfpAfSPjbz+1N8NhQLwcSabKHSncgHvZoiAZpjIZojIVojAZpiIZojPn7Jm4HC8f4xzXGQkQiDVi00Z/d7VBSgxPDvH+b/6zx/u3Q1wNbfw3DBw5VUxBrnhTgcyaGeXzuaMgHcumi6lJE5EgcNqidc1kzez/+aG4P+IZzbqOZXV94/5YZLqOUUaDQam6Oh1k8t66oc5xzDKay9BbC/cDQWLAPJLP0D2f8dTJDfzLDvoQf8v2F914s5AHCwQBz4mGa4yH/2nuh1d4Sn7g9p66NluaFtCwMUxf2Dr4Gn8v4D1sZ2guJvTC0z19Gtwv79z8HPY/4+ybfCgesAvhNoz+grq69MLDuRbZ1PV1ESlBU/55zbg2wZtK+KQPaOXfN0RdLqpmZ0VBoGRfbch/hnCOZydOfzDCQzNA3XAj0Qrj3Dfvbfjd9ht6hNE/u7B/9Q8AdIuNDnh0U5M3xkRZ9hKbYiTTGTqGpOUTjvCBNMf+9plhorKveOb9LfUKg72XL+oc5qb0eBnf7y+4n/Ae2TNX9Dv6jUusnh3ibvz9cB5F6fx1umLgdrvNDfiZGzefzkEtBNgnZFJjn9x5ohL5I2WlmMqkoZkYs7BELe3Q0Rks6d6SbfqQVP3KtvXdcqI902z+ze5C+4Qx9w5mDJqaZLB72aIyGCuFdCPFomMbYCTTGTmZXfj6nzzmVunlB4mGP+kiQeCRIvZelPtdLXXof0fQ+QkN7ILEbBvf4I+cTe/x54Af3QOoQoT5ZIDhFiNf7S6QeQjH/Frxs0l9y6bHwHV3G7y+s85mDf1a0GdqXQ/uywvol0PYSPcpV5BhTUMusMb6bvhTJTI7+Qmj3Jwvr4bHW++T923uTPJUcoK/Qyge44+kNh/05Ia+Nush86sJ+oMcjQeqiHvHGIE2hHK3hFHOCWVqCKZq8NI2BJI1einqSxBkm5pJE8kOEckNYOgHpQf9afHrQb+GnBiAz7N+rPv5Wt2DUX6LNB++ffFtcMOqfn8vA3qdh95Ow4XuQ/MbYL1LX7of26LLcHy8QbSqp3kWkOApqqXnRkEc05NFeYgseIJd3/OyX3bz03PNJpHMkUlkSqSxD6RyJdJahVGGdzjGYyjKUypJI5xhKZ0mk/PW23mE2pbIMpsZfow8C9YVlIi9gNESD/hLxW/kNdSEaWv0BeCPd+s3xsev2zYWR+VNeqz8c52Bgp9+lv+cpf737SXjsPybOdNe4qND6HgnvZdB6sj/L3bHsQnfOHyjYt9UfONi7dWw71gKLzvGX1pM1o55UBQW1yFHwAkYsaEcU8lMZuUY/kMz4g+uSmQkD8AYKA/D87bH9W/cPjb1OHTzgbUTYC9AUD9ESD43dWlfohRjb9q/PR0MeYS9AJBQg7DURaX85kQWrCAcDhIMBPJwfgLufnBjiz93vX+8eraSI/6CYupHR8m1jo+dHtwsj6uvm+t34Lxbsuaw/3/2EEC4E8ci+yVPlBqP+JDmJvfDov/v7ok2w8Oyx4F54lj+6X6TCKKhFKsj4a/TtR/hE00wuT9+wf03+wFCGAxNupxvZ72+P3FLXO5Qmc6jZcA4hGDAihdCOBFcSDp5JJBgg2ug43nZxknuBRW4Xc6yfZtdPU6KX+v4d1GefIJY5QCifnPJznRfFxVuxurlYfRvE5/KSndvh2c/7QTywHdykcQXxVmg6zm8lL4zXlwUAAAzWSURBVLkQmhb5r5sW+Y+Mjbf64Z/Pw95N/ij+nkeg57fwqy8wOodT6ymF4C4EePvyku6pF5kJ+gaKzDIhL8Dc+ghz64u/DWz8ZDi9Q36rPZXNkc7mSY1b0oVl/HtTve7PncjDmeP5VTbHUDo32uWfSGVHb7+LkaTVBphDP61WWOhnTraf1vQArb19zA1sYa6tJY+xzutgf2gp/Q0XMBjtYCi2gFTdAnINi4jE66mLBKmPeNRHQtRF/EF99eEg9bkgdekc8bCHBQKF7vllcNY7/F8+NeA/VnYkuJ/5Gaz7dqEy47DgrLHgXnSO/0AakWNIQS0ikybDmdmflc7mGSpctx+5Vj/+ev5wOkdfOseOcdfzn31+G/UtrSRS/rX+wWSWRJ9/XX8w1XPI2/Im/o6MDeQLe8TDQeoiHrFwkLpwPfHwRcTjryW+IsC8/C5OHH6CBYMbaO9bT9MLNxNw/sj4TP1Ccg0LsEgjFm0gEG3EizViI4+JnbA0Fkbkj9unZ8VLiRTUInJM+de4wzSXcIt9d/deurrOnvI95xzDmXEBPhLmhYF947dHBvCN/yOhbzjDjt7hsX3pXOGWvRMLyx8SIc1p9nvODGxmZd+ztPb1U2/7qGeYBhumnmHilpqyfJOlLEbai5P26kiH6smGGsiFGshHGnGRJog2Eog148WaCMabCNY1E6lrIdowh3Bdsz8rn8K+piioRaSqmRnxcJB4OEh7w/R8ZjaXZyiTm9DKT6SyDGX89Y6M392fzORJZnJ+l386hUv6t8tZup9AehAvM4iXSRDMDhLOJgjlEkRyQ0RzCWKZBLHhIRrZRwM9NNgQjQwVFfiDxBiyOAmrJ+nVkQ7ESQfryQTj5IJ15EINuHAdLlwP4Qa/5R9pwIs1EIw3EYo1EIo3EYvVEQ0HiYU9UlmHc+7guwLyOf/++1zKv+9+wjpVuCd/3NoC/qC82JzCukV/WBwlBbWIyCRBL0CjF6AxGprRn+OcI5XNj7bmt6VzDCWTpAd7yQz1kh3qJT/URz7Z5890l+onkBogmOkjmBkknBkgnBsknutnTmYnUTdE3A0TI0ngoIccHizrAiSIMkiMk12YnvuzRCxLhAxhsoRJ403xtLqSfkfMH2Efn4PF5hTmyh8X5KPbk/Z7Yf+PhFy6sGSK285nJu4PxaFhHtTP88cXRKbpr7ljSEEtIlImZjZ6H/+cupGJehqAtqP74HweMgkyw/2kEn2kE31khvrJDPeTHe4nPzxAPuUvlhrA0oMM9u4lFG8i6YIM50MM5T2G80ESOY9ENsBgzmMw69GfDZB2QdKESFNYu7FtI0+LDdLCIC02QIsN0Dw4yJzEgL/fNjGHAZoZeNHegzxW1B8bpcp4MYbDc0lG20hG28jE2snE28nG28nXdZCva8fVz8Orm0Mo6BHyAkSC/hINe0SDHiHPSp+P4CgoqEVEZptAACINhCINhJoXFnVKsY+5zOcdiXR2dEzAyBiAwWSWgZR/mSCXd+SdI5t35PKO/rzjQGE7O/JezhHIJQmle4lmCku2j1imj1i2Dy+fJk2IDEHSLkgGj5Tz/D8IXJC080jlg6Tc2P5k3iOdD5B0QVJ5j+F8gGh+iJb8fubkD9BuvbRne2lPH6A90Usb6+iwXurt4FsFUy7IHprZ45p51jWTIoTDCn86GIFAAC/gr/3tAIGAFdb+a290fwDP85cjoaAWEZGiBQJjD92himaNdc6RyTnSuTyZbJ5Mzr+dcHcuz/bkIAzsxA3sJDC4i8DQLoKJ3QSHdrNoeDcnJfdiuTTO5cH51/KdczicPxOec7hsHvD3+7chjF8D42K+VApqERGZ9cyMcNAIBwNw0BQD9cC8Gf35+bw/HoG/Lj12NdGtiIjIDAsE/FkHj+jcaS6LiIiITCMFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVDAFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVDAFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVDAFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVDAFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVLCigtrMXmtmT5vZZjP7+BTvv93MHi8sD5rZyukvqoiISO05bFCbmQfcDKwGlgNXmdnySYc9B1zgnDsd+Cxw63QXVEREpBYV06I+F9jsnNvinEsD3wUuH3+Ac+5B59yBwstfA4umt5giIiK1KVjEMQuBreNe9wDnvcjx7wJ+PNUbZnYdcB1AW1sb3d3dxZVSjsjg4KDq+BhQPc881fHMUx1XrmKC2qbY56Y80OxV+EH9iqned87dSqFbfOnSpa6rq6u4UsoR6e7uRnU881TPM091PPNUx5WrmKDuAY4b93oRsH3yQWZ2OvB1YLVzbt/0FE9ERKS2FXON+hHgFDNbbGZh4ErgnvEHmNnxwN3AO5xzm6a/mCIiIrXpsC1q51zWzN4P/BTwgG845zaa2fWF928BPgm0Al8xM4Csc+7smSu2iIhIbSim6xvn3BpgzaR9t4zbfjfw7uktmoiIiGhmMhERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKpqAWERGpYApqERGRCqagFhERqWAKahERkQqmoBYREalgCmoREZEKVlRQm9lrzexpM9tsZh+f4n0zs38uvP+4mZ01/UUVERGpPYcNajPzgJuB1cBy4CozWz7psNXAKYXlOuCr01xOERGRmlRMi/pcYLNzbotzLg18F7h80jGXA7c736+BZjObP81lFRERqTnFBPVCYOu41z2FfaUeIyIiIiUKFnGMTbHPHcExmNl1+F3jACkz21DEz5cjNxfYW+5C1ADV88xTHc881fGxsbTUE4oJ6h7guHGvFwHbj+AYnHO3ArcCmNlvnXNnl1RaKYnq+NhQPc881fHMUx0fG2b221LPKabr+xHgFDNbbGZh4ErgnknH3AP8SWH098uAPufcjlILIyIiIhMdtkXtnMua2fuBnwIe8A3n3EYzu77w/i3AGuBSYDMwBFw7c0UWERGpHcV0feOcW4MfxuP33TJu2wHvK/Fn31ri8VI61fGxoXqeearjmac6PjZKrmfzM1ZEREQqkaYQFRERqWBlCerDTUkqR8/Mfm9m681s7ZGMMpSpmdk3zGz3+FsLzWyOmf3czJ4prFvKWcZqd4g6/rSZbSt8n9ea2aXlLGO1M7PjzOxeM3vSzDaa2YcK+/VdniYvUsclf5ePedd3YUrSTcBF+Ld1PQJc5Zx74pgWZJYzs98DZzvndF/kNDKzVcAg/kx8Kwr7vgjsd879XeEPzxbn3A3lLGc1O0QdfxoYdM79fTnLNlsUZo6c75x7zMwagEeBNwDXoO/ytHiROv4jSvwul6NFXcyUpCIVyTl3H7B/0u7LgW8Wtr+J/z+jHKFD1LFMI+fcDufcY4XtAeBJ/Nkk9V2eJi9SxyUrR1BrutFjwwE/M7NHCzPCyczpGJk3oLBuL3N5Zqv3F57O9w11yU4fMzsROBN4GH2XZ8SkOoYSv8vlCOqiphuVo/Zy59xZ+E82e1+hO1GkWn0VWAKcAewAvlTe4swOZlYPfA/4sHOuv9zlmY2mqOOSv8vlCOqiphuVo+Oc215Y7wb+G/+Sg8yMXSNPiyusd5e5PLOOc26Xcy7nnMsD/4q+z0fNzEL4AfIt59zdhd36Lk+jqer4SL7L5QjqYqYklaNgZnWFwQuYWR1wMaAHoMyce4CrC9tXAz8oY1lmpUmPzb0CfZ+PipkZ8G/Ak865fxj3lr7L0+RQdXwk3+WyTHhSGI7+j4xNSfq3x7wQs5iZnYTfigZ/9rlvq46nh5l9B+jCf9LQLuBTwPeBO4HjgReAtzjnNBjqCB2ijrvwuwod8HvgT/U8gSNnZq8A7gfWA/nC7k/gX0PVd3kavEgdX0WJ32XNTCYiIlLBNDOZiIhIBVNQi4iIVDAFtYiISAVTUIuIiFQwBbWIiEgFU1CLiIhUMAW1iIhIBVNQi4iIVLD/D4Gf7T51ERAFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_learning_curves(history_cus_subclass)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### lambda方式自定义层次\n",
    "对于没有参数的函数，进行自定义层次，因为层次比较小，所以使用lambda方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([4.5417706e-05 6.7153489e-03 6.9314718e-01 5.0067153e+00 1.0000046e+01], shape=(5,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 自定义一个softplus层次\n",
    "# tf.nn.softplus : log(1+e^x)\n",
    "customized_softplus = keras.layers.Lambda(lambda x : tf.nn.softplus(x))\n",
    "print(customized_softplus([-10., -5., 0., 5., 10.]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "customized_dense_layer_2 (Cu (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "customized_dense_layer_3 (Cu (None, 1)                 31        \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 构建模型\n",
    "model = keras.models.Sequential([\n",
    "    CustomizedDenseLayer(30, activation='relu',input_shape=x_train.shape[1:]),\n",
    "    CustomizedDenseLayer(1),\n",
    "    customized_softplus\n",
    "])\n",
    "model.summary()\n",
    "\n",
    "# 模型编译，固化模型\n",
    "model.compile(loss='mean_squared_error',optimizer='adam')\n",
    "\n",
    "# 回调函数\n",
    "callbacks = [keras.callbacks.EarlyStopping(patience=5,min_delta=1e-2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
